Gabe Black has uploaded this change for review. (
https://gem5-review.googlesource.com/c/public/gem5/+/48363 )
Change subject: scons: Update the special module importer API.
......................................................................
scons: Update the special module importer API.
In the SConscript, there is a special importer which enables importing
embedded code using various m5.* paths. This was implemented using an
API which has been deprecated and replaced in more recent versions of
python.
Change-Id: I5900f269af48befbcedcb9d25353f04f6297ce9d
---
M src/SConscript
M src/python/importer.py
M src/python/m5/objects/__init__.py
3 files changed, 79 insertions(+), 75 deletions(-)
diff --git a/src/SConscript b/src/SConscript
index dbd28ea..34ba841 100644
--- a/src/SConscript
+++ b/src/SConscript
@@ -42,6 +42,9 @@
import distutils.spawn
import functools
import imp
+import importlib
+import importlib.machinery
+import importlib.util
import os
import os.path
import re
@@ -767,10 +770,20 @@
#
SimObject.fixed = True
-class DictImporter(object):
- '''This importer takes a dictionary of arbitrary module names that
- map to arbitrary filenames.'''
+class SimpleModuleLoader(importlib.abc.Loader):
+ '''A simple wrapper which delegates setting up a module to a
function.'''
+ def __init__(self, executor):
+ super(SimpleModuleLoader, self).__init__()
+ self.executor = executor
+ def create_module(self, spec):
+ return None
+
+ def exec_module(self, module):
+ self.executor(module)
+
+class M5MetaPathFinder(importlib.abc.MetaPathFinder):
def __init__(self, modules):
+ super(M5MetaPathFinder, self).__init__()
self.modules = modules
self.installed = set()
@@ -780,42 +793,46 @@
del sys.modules[module]
self.installed = set()
- def find_module(self, fullname, path):
- if fullname == 'm5.defines':
- return self
+ def find_spec(self, fullname, path, target=None):
+ spec = None
- if fullname == 'm5.objects':
- return self
+ # If this isn't even in the m5 package, ignore it.
+ if fullname.startswith('m5.'):
+ if fullname.startswith('m5.objects'):
+ # When imported in this context, return a spec for a dummy
+ # package which just serves to house the modules within it.
+ # This is subtley different from "import * from m5.objects"
+ # which relies on the __init__.py in m5.objects. That in
turn
+ # indirectly relies on the c++ based _m5 package which
doesn't
+ # exist yet.
+ if fullname == 'm5.objects':
+ dummy_loader = SimpleModuleLoader(lambda x: None)
+ spec = importlib.machinery.ModuleSpec(
+ name=fullname, loader=dummy_loader,
+ is_package=True)
+ spec.loader_state = self.modules.keys()
- source = self.modules.get(fullname, None)
- if source is not None and fullname.startswith('m5.objects'):
- return self
+ # If this is a module within the m5.objects package,
return a
+ # spec that maps to its source file.
+ elif fullname in self.modules:
+ source = self.modules[fullname]
+ spec = importlib.util.spec_from_file_location(
+ name=fullname, location=source.abspath)
- return None
+ # The artificial m5.defines subpackage.
+ elif fullname == 'm5.defines':
+ def build_m5_defines(module):
+ module.__dict__['buildEnv'] = dict(build_env)
- def load_module(self, fullname):
- mod = imp.new_module(fullname)
- sys.modules[fullname] = mod
- self.installed.add(fullname)
+ spec = importlib.util.spec_from_loader(name=fullname,
+ loader=SimpleModuleLoader(build_m5_defines))
- mod.__loader__ = self
- if fullname == 'm5.objects':
- mod.__path__ = fullname.split('.')
- return mod
+ # If we're handling this module, write it down so we can unload it
+ # later.
+ if spec is not None:
+ self.installed.add(fullname)
- if fullname == 'm5.defines':
- mod.__dict__['buildEnv'] = dict(build_env)
- return mod
-
- source = self.modules[fullname]
- if source.modname == '__init__':
- mod.__path__ = source.modpath
- mod.__file__ = source.abspath
-
- compiled = compile(open(source.abspath).read(),
source.abspath, 'exec')
- exec(compiled, mod.__dict__)
-
- return mod
+ return spec
import m5.SimObject
import m5.params
@@ -826,7 +843,7 @@
# install the python importer so we can grab stuff from the source
# tree itself. We can't have SimObjects added after this point or
# else we won't know about them for the rest of the stuff.
-importer = DictImporter(PySource.modules)
+importer = M5MetaPathFinder(PySource.modules)
sys.meta_path[0:0] = [ importer ]
# import all sim objects so we can populate the all_objects list
diff --git a/src/python/importer.py b/src/python/importer.py
index b89b4a8..4e5e907 100644
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -24,12 +24,25 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import importlib
+import os
+
+class ByteCodeLoader(importlib.abc.Loader):
+ def __init__(self, code):
+ super(ByteCodeLoader, self).__init__()
+ self.code = code
+
+ def exec_module(self, module):
+ exec(self.code, module.__dict__)
+
# Simple importer that allows python to import data from a dict of
# code objects. The keys are the module path, and the items are the
# filename and bytecode of the file.
class CodeImporter(object):
def __init__(self):
self.modules = {}
+ override_var = os.environ.get('M5_OVERRIDE_PY_SOURCE', 'false')
+ self.override = (override_var.lower() in ('true', 'yes'))
def add_module(self, filename, abspath, modpath, code):
if modpath in self.modules:
@@ -37,50 +50,24 @@
self.modules[modpath] = (filename, abspath, code)
- def find_module(self, fullname, path):
- if fullname in self.modules:
- return self
+ def find_spec(self, fullname, path, target=None):
+ if fullname not in self.modules:
+ return None
- return None
+ srcfile, abspath, code = self.modules[fullname]
- def load_module(self, fullname):
- # Because the importer is created and initialized in its own
- # little sandbox (in init.cc), the globals that were available
- # when the importer module was loaded and CodeImporter was
- # defined are not available when load_module is actually
- # called. Soooo, the imports must live here.
- import imp
- import os
- import sys
+ if self.override and os.path.exists(abspath):
+ src = open(abspath, 'r').read()
+ code = compile(src, abspath, 'exec')
- try:
- mod = sys.modules[fullname]
- except KeyError:
- mod = imp.new_module(fullname)
- sys.modules[fullname] = mod
+ is_package = (os.path.basename(srcfile) == '__init__.py')
+ spec = importlib.util.spec_from_loader(
+ name=fullname, loader=ByteCodeLoader(code),
+ is_package=is_package)
- try:
- mod.__loader__ = self
- srcfile,abspath,code = self.modules[fullname]
+ spec.loader_state = self.modules.keys()
- override =
os.environ.get('M5_OVERRIDE_PY_SOURCE', 'false').lower()
- if override in ('true', 'yes') and os.path.exists(abspath):
- src = open(abspath, 'r').read()
- code = compile(src, abspath, 'exec')
-
- if os.path.basename(srcfile) == '__init__.py':
- mod.__path__ = fullname.split('.')
- mod.__package__ = fullname
- else:
- mod.__package__ = fullname.rpartition('.')[0]
- mod.__file__ = srcfile
-
- exec(code, mod.__dict__)
- except Exception:
- del sys.modules[fullname]
- raise
-
- return mod
+ return spec
# Create an importer and add it to the meta_path so future imports can
# use it. There's currently nothing in the importer, but calls to
diff --git a/src/python/m5/objects/__init__.py
b/src/python/m5/objects/__init__.py
index 3ec3b8c..e59f9a8 100644
--- a/src/python/m5/objects/__init__.py
+++ b/src/python/m5/objects/__init__.py
@@ -28,10 +28,10 @@
from m5.SimObject import *
try:
- modules = __loader__.modules
+ modules = __spec__.loader_state
except NameError:
modules = { }
-for module in modules.keys():
+for module in modules:
if module.startswith('m5.objects.'):
exec("from %s import *" % module)
--
To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/48363
To unsubscribe, or for help writing mail filters, visit
https://gem5-review.googlesource.com/settings
Gerrit-Project: public/gem5
Gerrit-Branch: develop
Gerrit-Change-Id: I5900f269af48befbcedcb9d25353f04f6297ce9d
Gerrit-Change-Number: 48363
Gerrit-PatchSet: 1
Gerrit-Owner: Gabe Black <[email protected]>
Gerrit-MessageType: newchange
_______________________________________________
gem5-dev mailing list -- [email protected]
To unsubscribe send an email to [email protected]
%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s
