https://github.com/python/cpython/commit/157f271de352f16ecd052fb1fb0fcce528962407 commit: 157f271de352f16ecd052fb1fb0fcce528962407 branch: main author: Filipe Laíns <[email protected]> committer: FFY00 <[email protected]> date: 2026-02-19T15:25:50Z summary:
gh-139899: Introduce MetaPathFinder.discover and PathEntryFinder.discover (#139900) * gh-139899: Introduce MetaPathFinder.discover and PathEntryFinder.discover Signed-off-by: Filipe Laíns <[email protected]> * Fix doc reference Signed-off-by: Filipe Laíns <[email protected]> * Remove specific doc references Signed-off-by: Filipe Laíns <[email protected]> * Fix docstrings Signed-off-by: Filipe Laíns <[email protected]> * Revert "Remove specific doc references" This reverts commit 31d1a8f5510e0f7a53016c7120ea2e1bda46e60c. Signed-off-by: Filipe Laíns <[email protected]> * Fix news references Signed-off-by: Filipe Laíns <[email protected]> * Add docs warning Signed-off-by: Filipe Laíns <[email protected]> * Raise ValueError on invalid parent Signed-off-by: Filipe Laíns <[email protected]> * Dedupe __path__ in PathFinder.discover Signed-off-by: Filipe Laíns <[email protected]> * Use context manager and add error handling to os.scandir Signed-off-by: Filipe Laíns <[email protected]> * Raise ValueError on invalid parent Signed-off-by: Filipe Laíns <[email protected]> * Dedupe when package exists with multiple suffixes Signed-off-by: Filipe Laíns <[email protected]> * Apply suggestions from code review Co-authored-by: Alyssa Coghlan <[email protected]> * Add tests Signed-off-by: Filipe Laíns <[email protected]> --------- Signed-off-by: Filipe Laíns <[email protected]> Co-authored-by: Alyssa Coghlan <[email protected]> Co-authored-by: Brett Cannon <[email protected]> files: A Lib/test/test_importlib/test_discover.py A Misc/NEWS.d/next/Library/2025-10-10-14-08-58.gh-issue-139899.09leRY.rst M Doc/library/importlib.rst M Lib/importlib/_bootstrap_external.py M Lib/importlib/abc.py diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 5f0858cb134ebf..795524e5b62145 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -275,6 +275,28 @@ ABC hierarchy:: .. versionchanged:: 3.4 Returns ``None`` when called instead of :data:`NotImplemented`. + .. method:: discover(parent=None) + + An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, :meth:`MetaPathFinder.discover` will + search for top-level modules. + + Returns an iterable of possible specs. + + Raises :exc:`ValueError` if *parent* is not a package module. + + .. warning:: + This method can potentially yield a very large number of objects, and + it may carry out IO operations when computing these values. + + Because of this, it will generaly be desirable to compute the result + values on-the-fly, as they are needed. As such, the returned object is + only guaranteed to be an :class:`iterable <collections.abc.Iterable>`, + instead of a :class:`list` or other + :class:`collection <collections.abc.Collection>` type. + + .. versionadded:: next + .. class:: PathEntryFinder @@ -307,6 +329,28 @@ ABC hierarchy:: :meth:`importlib.machinery.PathFinder.invalidate_caches` when invalidating the caches of all cached finders. + .. method:: discover(parent=None) + + An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, :meth:`PathEntryFinder.discover` will + search for top-level modules. + + Returns an iterable of possible specs. + + Raises :exc:`ValueError` if *parent* is not a package module. + + .. warning:: + This method can potentially yield a very large number of objects, and + it may carry out IO operations when computing these values. + + Because of this, it will generaly be desirable to compute the result + values on-the-fly, as they are needed. As such, the returned object is + only guaranteed to be an :class:`iterable <collections.abc.Iterable>`, + instead of a :class:`list` or other + :class:`collection <collections.abc.Collection>` type. + + .. versionadded:: next + .. class:: Loader diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index b576ceb1ce9f6e..213190d2098e75 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1283,6 +1283,23 @@ def find_spec(cls, fullname, path=None, target=None): else: return spec + @classmethod + def discover(cls, parent=None): + if parent is None: + path = sys.path + elif parent.submodule_search_locations is None: + raise ValueError(f'{parent} is not a package module') + else: + path = parent.submodule_search_locations + + for entry in set(path): + if not isinstance(entry, str): + continue + if (finder := cls._path_importer_cache(entry)) is None: + continue + if discover := getattr(finder, 'discover', None): + yield from discover(parent) + @staticmethod def find_distributions(*args, **kwargs): """ @@ -1432,6 +1449,37 @@ def path_hook_for_FileFinder(path): return path_hook_for_FileFinder + def _find_children(self): + with _os.scandir(self.path) as scan_iterator: + while True: + try: + entry = next(scan_iterator) + if entry.name == _PYCACHE: + continue + # packages + if entry.is_dir() and '.' not in entry.name: + yield entry.name + # files + if entry.is_file(): + yield from { + entry.name.removesuffix(suffix) + for suffix, _ in self._loaders + if entry.name.endswith(suffix) + } + except OSError: + pass # ignore exceptions from next(scan_iterator) and os.DirEntry + except StopIteration: + break + + def discover(self, parent=None): + if parent and parent.submodule_search_locations is None: + raise ValueError(f'{parent} is not a package module') + + module_prefix = f'{parent.name}.' if parent else '' + for child_name in self._find_children(): + if spec := self.find_spec(module_prefix + child_name): + yield spec + def __repr__(self): return f'FileFinder({self.path!r})' diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 87922f32d1111b..9ca127ad9c7d0f 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -45,6 +45,16 @@ def invalidate_caches(self): This method is used by importlib.invalidate_caches(). """ + def discover(self, parent=None): + """An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, MetaPathFinder.discover will search + for top-level modules. + + Returns an iterable of possible specs. + """ + return () + + _register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter, machinery.PathFinder, machinery.WindowsRegistryFinder) @@ -58,6 +68,15 @@ def invalidate_caches(self): This method is used by PathFinder.invalidate_caches(). """ + def discover(self, parent=None): + """An optional method which searches for possible specs with given + *parent* module spec. If *parent* is *None*, PathEntryFinder.discover + will search for top-level modules. + + Returns an iterable of possible specs. + """ + return () + _register(PathEntryFinder, machinery.FileFinder) diff --git a/Lib/test/test_importlib/test_discover.py b/Lib/test/test_importlib/test_discover.py new file mode 100644 index 00000000000000..8c5fa65a564c6d --- /dev/null +++ b/Lib/test/test_importlib/test_discover.py @@ -0,0 +1,121 @@ +from unittest.mock import Mock + +from test.test_importlib import util + +importlib = util.import_importlib('importlib') +machinery = util.import_importlib('importlib.machinery') + + +class DiscoverableFinder: + def __init__(self, discover=[]): + self._discovered_values = discover + + def find_spec(self, fullname, path=None, target=None): + raise NotImplemented + + def discover(self, parent=None): + yield from self._discovered_values + + +class TestPathFinder: + """PathFinder implements MetaPathFinder, which uses the PathEntryFinder(s) + registered in sys.path_hooks (and sys.path_importer_cache) to search + sys.path or the parent's __path__. + + PathFinder.discover() should redirect to the .discover() method of the + PathEntryFinder for each path entry. + """ + + def test_search_path_hooks_top_level(self): + modules = [ + self.machinery.ModuleSpec(name='example1', loader=None), + self.machinery.ModuleSpec(name='example2', loader=None), + self.machinery.ModuleSpec(name='example3', loader=None), + ] + + with util.import_state( + path_importer_cache={ + 'discoverable': DiscoverableFinder(discover=modules), + }, + path=['discoverable'], + ): + discovered = list(self.machinery.PathFinder.discover()) + + self.assertEqual(discovered, modules) + + + def test_search_path_hooks_parent(self): + parent = self.machinery.ModuleSpec(name='example', loader=None, is_package=True) + parent.submodule_search_locations.append('discoverable') + + children = [ + self.machinery.ModuleSpec(name='example.child1', loader=None), + self.machinery.ModuleSpec(name='example.child2', loader=None), + self.machinery.ModuleSpec(name='example.child3', loader=None), + ] + + with util.import_state( + path_importer_cache={ + 'discoverable': DiscoverableFinder(discover=children) + }, + path=[], + ): + discovered = list(self.machinery.PathFinder.discover(parent)) + + self.assertEqual(discovered, children) + + def test_invalid_parent(self): + parent = self.machinery.ModuleSpec(name='example', loader=None) + with self.assertRaises(ValueError): + list(self.machinery.PathFinder.discover(parent)) + + +( + Frozen_TestPathFinder, + Source_TestPathFinder, +) = util.test_both(TestPathFinder, importlib=importlib, machinery=machinery) + + +class TestFileFinder: + """FileFinder implements PathEntryFinder and provides the base finder + implementation to search the file system. + """ + + def get_finder(self, path): + loader_details = [ + (self.machinery.SourceFileLoader, self.machinery.SOURCE_SUFFIXES), + (self.machinery.SourcelessFileLoader, self.machinery.BYTECODE_SUFFIXES), + ] + return self.machinery.FileFinder(path, *loader_details) + + def test_discover_top_level(self): + modules = {'example1', 'example2', 'example3'} + with util.create_modules(*modules) as mapping: + finder = self.get_finder(mapping['.root']) + discovered = list(finder.discover()) + self.assertEqual({spec.name for spec in discovered}, modules) + + def test_discover_parent(self): + modules = { + 'example.child1', + 'example.child2', + 'example.child3', + } + with util.create_modules(*modules) as mapping: + example = self.get_finder(mapping['.root']).find_spec('example') + finder = self.get_finder(example.submodule_search_locations[0]) + discovered = list(finder.discover(example)) + self.assertEqual({spec.name for spec in discovered}, modules) + + def test_invalid_parent(self): + with util.create_modules('example') as mapping: + finder = self.get_finder(mapping['.root']) + example = finder.find_spec('example') + with self.assertRaises(ValueError): + list(finder.discover(example)) + + +( + Frozen_TestFileFinder, + Source_TestFileFinder, +) = util.test_both(TestFileFinder, importlib=importlib, machinery=machinery) diff --git a/Misc/NEWS.d/next/Library/2025-10-10-14-08-58.gh-issue-139899.09leRY.rst b/Misc/NEWS.d/next/Library/2025-10-10-14-08-58.gh-issue-139899.09leRY.rst new file mode 100644 index 00000000000000..fe5e7d17ab6c8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-10-14-08-58.gh-issue-139899.09leRY.rst @@ -0,0 +1,3 @@ +Introduced :meth:`importlib.abc.MetaPathFinder.discover` +and :meth:`importlib.abc.PathEntryFinder.discover` to allow module and submodule +name discovery without assuming the use of traditional filesystem based imports. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
