How can I make inspect package to alter caller locals

Question

I am trying to write a save/load command like the one in MATLAB (ability to save local variables to disk or load them into current context, or work space in MATLAB's terminology).

I wrote the following code, but it doesn't seem to work, as the variables in the outer scope are not replaced, probability because of a memory copy which takes place somewhere.

Here is the code:

import shelve
import logging
import inspect

logger = logging.getLogger()
def save_locals(filename, keys=None):
    my_shelf = shelve.open(filename, 'n')  # 'n' for new
    caller_locals = inspect.stack()[1][0].f_locals
    if keys is None:
        keys = caller_locals.keys()
    for key in keys:
        try:
            my_shelf[key] = caller_locals[key]
        except TypeError:
            #
            # __builtins__, my_shelf, and imported modules can not be shelved.
            #
            print('ERROR shelving: {0}'.format(key))
    my_shelf.close()


def load_locals(filename, keys=None):
    my_shelf = shelve.open(filename)
    caller_locals = inspect.stack()[1][0].f_locals
    if keys is None:
        keys = list(my_shelf.keys())
    for key in keys:
        try:
            caller_locals[key] = my_shelf[key]
        except ValueError:
            print('cannot get variable %s'.format(key))

Here is the test which fails:

from unittest import TestCase
from .io import save_locals, load_locals

class TestIo(TestCase):
    def test_save_load(self):
        sanity = 'sanity'
        an_int = 3
        a_float = 3.14
        a_list = [1, 2, 3]
        a_dict = [{'a': 5, 'b': 3}]
        save_locals('temp')
        an_int = None
        a_float = None
        a_list = None
        a_dict = None
        load_locals('temp')
        self.assertIn('an_int', locals())
        self.assertIn('a_float', locals())
        self.assertIn('a_list', locals())
        self.assertIn('a_dict', locals())
        self.assertEqual(an_int, 3)
        self.assertEqual(a_float, 3.14)
        self.assertEqual(a_list, [1, 2, 3])
        self.assertEqual(a_dict, [{'a': 5, 'b': 3}])

When I break-point inside load_locals I can see it changes the f_locals dictionary but when the function returns they do not change.

Answer 1

No, you can't update local variables on the fly. The reason is because the local symbol table is saved as a C array for optimization and both locals() and frame.f_locals end up returning a copy to that local symbol table. The official response is that modifying locals() has undefined behavior. This thread talks a bit about it.

It ends up being extra weird because calling locals() or frame.f_locals returns the same dictionary each time, which gets re-synced at different times. Here just calling frame.f_locals resets the local

def test_locals():
    frame = inspect.stack()[1][0]
    caller_locals = frame.f_locals
    caller_locals['an_int'] = 5
    print(caller_locals)
    _ = frame.f_locals
    print(caller_locals)


def call_test_locals():
    an_int = 3
    test_locals()


call_test_locals()

output:

{'an_int': 5}
{'an_int': 3}

The behavior is going to depend on the Python implementation and probably other edge cases, but a few examples where (1) the variable is defined and is not updated; (2) the variable is not defined and is updated; (3) the variable is defined and subsequently deleted and is not updated.

def test_locals():
    frame = inspect.stack()[1][0]
    caller_locals = frame.f_locals
    caller_locals['an_int'] = 5


def call_test_locals1():
    an_int = 3
    print('calling', locals())
    test_locals()
    print('done', locals())


def call_test_locals2():
    print('calling', locals())
    test_locals()
    print('done', locals())


def call_test_locals3():
    an_int = 3
    del an_int
    print('calling', locals())
    test_locals()
    print('done', locals())


print('\n1:')
call_test_locals1()
print('\n2:')
call_test_locals2()
print('\n3:')
call_test_locals3()

output:

1:
calling {'an_int': 3}
done {'an_int': 3}

2:
calling {}
done {'an_int': 5}

3:
calling {}
done {}

If you're running Python 2, you could use exec to execute a string into the local namespace, but it won't work in Python 3 and is in general probably a bad idea.

import shelve
import logging
import inspect

logger = logging.getLogger()
def save_locals(filename, keys=None):
    my_shelf = shelve.open(filename, 'n')  # 'n' for new
    caller_locals = inspect.stack()[1][0].f_locals
    if keys is None:
        keys = caller_locals.keys()
    for key in keys:
        try:
            my_shelf[key] = caller_locals[key]
        except TypeError:
            #
            # __builtins__, my_shelf, and imported modules can not be shelved.
            #
            print('ERROR shelving: {0}'.format(key))
    my_shelf.close()



def load_locals_string(filename, keys=None):
    my_shelf = shelve.open(filename)
    if keys is None:
        keys = list(my_shelf.keys())
    return ';'.join('{}={!r}'.format(key, my_shelf[key]) for key in keys)

and

from unittest import TestCase
from .io import save_locals, load_locals

class TestIo(TestCase):
    def test_save_load(self):
        sanity = 'sanity'
        an_int = 3
        a_float = 3.14
        a_list = [1, 2, 3]
        a_dict = [{'a': 5, 'b': 3}]
        save_locals('temp')
        an_int = None
        a_float = None
        a_list = None
        a_dict = None
        exec load_locals_string('temp')
        self.assertIn('an_int', locals())
        self.assertIn('a_float', locals())
        self.assertIn('a_list', locals())
        self.assertIn('a_dict', locals())
        self.assertEqual(an_int, 3)
        self.assertEqual(a_float, 3.14)
        self.assertEqual(a_list, [1, 2, 3])
        self.assertEqual(a_dict, [{'a': 5, 'b': 3}])

In Python 2, exec uses PyFrame_LocalsToFast to copy the variables back to the local scope, but can't in Python 3 because exec is a function. Martijn Pieters has a good post about it.