Dynamically import module from memory in Python 3 using Hooks

Question

What I want to achieve is exactly what this this answer proposes, however in Python 3.

The code below works fine in Python 2:

import sys
import imp
modules = {
"my_module":
"""class Test:
 def __init__(self):
 self.x = 5
 def print_number(self):
 print self.x"""} 
class StringImporter(object):
 def __init__(self, modules):
 self._modules = dict(modules)
 def find_module(self, fullname, path):
 if fullname in self._modules.keys():
 return self
 return None
 def load_module(self, fullname):
 if not fullname in self._modules.keys():
 raise ImportError(fullname)
 new_module = imp.new_module(fullname)
 exec self._modules[fullname] in new_module.__dict__
 return new_module
if __name__ == '__main__':
 sys.meta_path.append(StringImporter(modules))
 from my_module import Test
 my_test = Test()
 my_test.print_number() # prints 5

However, when making the obvious changes to Python 3 (enclosing exec and print in parentheses) I get the following code:

import sys
import imp
modules = {
"my_module":
"""class Test:
 def __init__(self):
 self.x = 5
 def print_number(self):
 print(self.x)"""} 
class StringImporter(object):
 def __init__(self, modules):
 self._modules = dict(modules)
 def find_module(self, fullname, path):
 if fullname in self._modules.keys():
 return self
 return None
 def load_module(self, fullname):
 if not fullname in self._modules.keys():
 raise ImportError(fullname)
 new_module = imp.new_module(fullname)
 exec(self._modules[fullname])
 return new_module
if __name__ == '__main__':
 sys.meta_path.append(StringImporter(modules))
 from my_module import Test
 my_test = Test()
 my_test.print_number() # Should print 5

Not that the exec() change was pretty significant. I didn't understand what that line did in Python 2, I "translated" it the way I think it's correct. However, the Python 3 code gives me the following error:

Traceback (most recent call last):
 File "main.py", line 35, in <module>
 from my_module import Test
 File "<frozen importlib._bootstrap>", line 991, in _find_and_load
 File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
 File "<frozen importlib._bootstrap>", line 655, in _load_unlocked
 File "<frozen importlib._bootstrap>", line 626, in _load_backward_compatible
KeyError: 'my_module'

What should I change in the code in order to work in Python 3 the exactly same way it works in Python 2?

Observation: This does not answer my question as I'm not interested in importing a module from .pyc.

Answer 1

The short answer is that you forgot to translate the latter half of the exec statement from the code sample. That causes the exec to be applied in the context of the load_module method — not the new_module; so specify the context:

exec(self._modules[fullname], new_module.__dict__)

However, using a Python versioned 3.4 or higher, you become subject to PEP 451 (the introduction of module specs), as well as the deprecation of the imp module, in favor of importlib. Particularly:

• The imp.new_module(name) function is replaced by importlib.util.module_from_spec(spec).
• An abstract base class for meta path finder objects is supplied: importlib.abc.MetaPathFinder.
• And such finder objects now use find_spec rather than find_module.

Here is a very close reimplementation of the code sample.

import importlib
import sys
import types
class StringLoader(importlib.abc.Loader):
 def __init__(self, modules):
 self._modules = modules
 def has_module(self, fullname):
 return (fullname in self._modules)
 def create_module(self, spec):
 if self.has_module(spec.name):
 module = types.ModuleType(spec.name)
 exec(self._modules[spec.name], module.__dict__)
 return module
 def exec_module(self, module):
 pass
class StringFinder(importlib.abc.MetaPathFinder):
 def __init__(self, loader):
 self._loader = loader
 def find_spec(self, fullname, path, target=None):
 if self._loader.has_module(fullname):
 return importlib.machinery.ModuleSpec(fullname, self._loader)
if __name__ == '__main__':
 modules = {
 'my_module': """
 BAZ = 42
 class Foo:
 def __init__(self, *args: str):
 self.args = args
 def bar(self):
 return ', '.join(self.args)
 """}
 finder = StringFinder(StringLoader(modules))
 sys.meta_path.append(finder)
 import my_module
 foo = my_module.Foo('Hello', 'World!')
 print(foo.bar())
 print(my_module.BAZ)