https://github.com/python/cpython/commit/c2eb1a85910b917d398768423b53e518c1e362da commit: c2eb1a85910b917d398768423b53e518c1e362da branch: 3.15 author: Miss Islington (bot) <[email protected]> committer: Yhg1s <[email protected]> date: 2026-10-02T19:04:35+02:00 summary:
[3.15] gh-155695: Remove resolved names from sys.lazy_modules more consistently (GH-157714) (#158607) gh-155695: Remove resolved names from sys.lazy_modules more consistently (GH-157714) (cherry picked from commit be87a85e2fb0cc22e6c4292ecec1bf7e3d0d3ace) Co-authored-by: Brittany Reynoso <[email protected]> Co-authored-by: Pablo Galindo Salgado <[email protected]> Co-authored-by: Petr Viktorin <[email protected]> files: A Lib/test/test_lazy_import/data/init_fails.py A Lib/test/test_lazy_import/data/lazy_on_init_fails.py A Misc/NEWS.d/next/Core_and_Builtins/2026-09-16-15-02-11.gh-issue-155695.Kf3xQa.rst M Include/internal/pycore_import.h M Lib/test/test_lazy_import/__init__.py M Objects/lazyimportobject.c M Python/import.c diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 81448e23787fb39..983ee0ad22789be 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -87,6 +87,8 @@ extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp); extern PyObject * _PyImport_InitLazyModules( PyInterpreterState *interp); extern void _PyImport_ClearLazyModules(PyInterpreterState *interp); +extern int _PyImport_DiscardLazyModule( + PyInterpreterState *interp, PyObject *name); extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp); extern int _PyImport_IsDefaultImportFunc( diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index d92020abc7a5658..8f5fe26728b7fb6 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -53,7 +53,8 @@ def test_sys_lazy_modules(self): self.fail('lazy import failed') self.assertFalse("test.test_lazy_import.data.basic2" in sys.modules) - self.assertIn("test.test_lazy_import.data", sys.lazy_modules) + # The package is already loaded, so it is not a pending import. + self.assertNotIn("test.test_lazy_import.data", sys.lazy_modules) self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) test.test_lazy_import.data.basic_from_unused.basic2 self.assertNotIn("test.test_import.data", sys.lazy_modules) @@ -1313,6 +1314,262 @@ def test_lazy_module_without_children_is_tracked(self): """) assert_python_ok("-c", code) + def test_already_loaded_module_is_not_tracked(self): + """A lazy import of an already loaded module should not be tracked.""" + code = textwrap.dedent(""" + import sys + + # Loaded by a regular import. + import json + lazy import json as lazy_json + assert "json" not in sys.lazy_modules, ( + f"expected 'json' not in sys.lazy_modules, got {sys.lazy_modules}" + ) + + # Loaded by reifying an earlier lazy import. + lazy import base64 + _ = base64.b64encode + lazy import base64 as lazy_base64 + assert "base64" not in sys.lazy_modules, ( + f"expected 'base64' not in sys.lazy_modules, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_already_loaded_submodule_is_not_tracked(self): + """`lazy from` a loaded submodule should not be tracked either.""" + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg.b + lazy from test.test_lazy_import.data.pkg import b + assert "test.test_lazy_import.data.pkg.b" not in sys.lazy_modules, ( + f"expected 'pkg.b' untracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_attribute_entry_removed_on_reification(self): + """`lazy from x import attr` should untrack "x.attr" once resolved.""" + code = textwrap.dedent(""" + import sys + lazy from test.test_lazy_import.data.basic2 import x + assert "test.test_lazy_import.data.basic2.x" in sys.lazy_modules, ( + f"expected 'basic2.x' tracked, got {sys.lazy_modules}" + ) + _ = x + assert "test.test_lazy_import.data.basic2.x" not in sys.lazy_modules, ( + f"expected 'basic2.x' untracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_already_loaded_attribute_is_not_tracked(self): + code = textwrap.dedent(""" + import sys + import math + + lazy from math import pi + assert "math.pi" not in sys.lazy_modules, sys.lazy_modules + assert pi == math.pi + """) + assert_python_ok("-c", code) + + def test_cached_lazy_attribute_tracks_its_target(self): + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.basic_from_unused + + holder = "test.test_lazy_import.data.basic_from_unused" + target = "test.test_lazy_import.data.basic2" + + namespace = {"__name__": "cached_import_test"} + exec(f"lazy from {holder} import basic2", namespace) + assert holder + ".basic2" not in sys.lazy_modules, sys.lazy_modules + assert target in sys.lazy_modules, sys.lazy_modules + exec("assert basic2.x == 42", namespace) + assert target not in sys.lazy_modules, sys.lazy_modules + """) + assert_python_ok("-c", code) + + def test_cached_attribute_keeps_pending_module_tracked(self): + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg as pkg + pkg.b = 42 + + lazy import test.test_lazy_import.data.pkg.b as pending + lazy from test.test_lazy_import.data.pkg import b + name = "test.test_lazy_import.data.pkg.b" + assert b == 42, b + assert name not in sys.modules, sys.modules + assert name in sys.lazy_modules, sys.lazy_modules + assert pending.foo() == "foo" + assert name not in sys.lazy_modules, sys.lazy_modules + """) + assert_python_ok("-c", code) + + def test_failed_reification_stays_tracked(self): + """A lazy import that fails to resolve must stay tracked.""" + code = textwrap.dedent(""" + import sys + lazy import test.test_lazy_import.data.broken_module + try: + _ = test.test_lazy_import.data.broken_module + except ValueError: + pass + else: + raise AssertionError("ValueError was not raised") + assert "test.test_lazy_import.data.broken_module" in sys.lazy_modules, ( + f"failed reification must stay tracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_blocked_module_is_still_tracked(self): + """A ``None`` entry in sys.modules must not count as loaded.""" + code = textwrap.dedent(""" + import sys + sys.modules['test.test_lazy_import.data.basic2'] = None + lazy import test.test_lazy_import.data.basic2 + assert "test.test_lazy_import.data.basic2" in sys.lazy_modules, ( + f"blocked module must stay tracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_initializing_module_is_still_tracked(self): + """A module that is still executing must not count as loaded.""" + code = textwrap.dedent(""" + import sys + name = "test.test_lazy_import.data.init_fails" + try: + import test.test_lazy_import.data.init_fails + except ValueError: + pass + else: + raise AssertionError("ValueError was not raised") + assert name not in sys.modules, "failed import left a module behind" + assert name in sys.lazy_modules, ( + f"expected {name!r} tracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_module_spec_descriptor_is_not_run(self): + """Checking whether a module is loaded must not run its descriptors.""" + code = textwrap.dedent(""" + import sys + import types + + class RaisingSpec(types.ModuleType): + @property + def __spec__(self): + raise RuntimeError("__spec__ descriptor was run") + + sys.modules["raising_spec"] = RaisingSpec("raising_spec") + lazy import raising_spec + assert "raising_spec" not in sys.lazy_modules, ( + f"expected 'raising_spec' untracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_spec_initializing_descriptor_is_not_run(self): + code = textwrap.dedent(""" + import sys + import types + + class Spec: + @property + def _initializing(self): + raise RuntimeError("_initializing descriptor was run") + + @property + def __dict__(self): + raise RuntimeError("__dict__ descriptor was run") + + class SlottedSpec: + __slots__ = () + + @property + def _initializing(self): + raise RuntimeError("_initializing descriptor was run") + + for spec in (Spec(), SlottedSpec()): + module = types.ModuleType("custom_spec") + module.__spec__ = spec + sys.modules["custom_spec"] = module + lazy import custom_spec + """) + assert_python_ok("-c", code) + + def test_cached_attribute_does_not_check_spec_twice(self): + code = textwrap.dedent(""" + import sys + import types + + class Spec: + def __init__(self): + self.calls = 0 + + @property + def _initializing(self): + self.calls += 1 + if self.calls == 2: + raise RuntimeError("_initializing was read twice") + return False + + module = types.ModuleType("cached_spec") + module.__spec__ = Spec() + module.attr = 1 + sys.modules["cached_spec"] = module + lazy from cached_spec import attr + assert attr == 1, attr + """) + assert_python_ok("-c", code) + + def test_spec_initializing_truth_conversion_is_not_run(self): + code = textwrap.dedent(""" + import sys + import types + from importlib.machinery import ModuleSpec + + class Flag: + def __bool__(self): + raise RuntimeError("_initializing truth conversion was run") + + released = memoryview(b"") + released.release() + for flag in (Flag(), released): + sys.lazy_modules.discard("custom_spec") + module = types.ModuleType("custom_spec") + module.__spec__ = ModuleSpec("custom_spec", None) + module.__spec__._initializing = flag + sys.modules["custom_spec"] = module + lazy import custom_spec + assert "custom_spec" in sys.lazy_modules, sys.lazy_modules + """) + assert_python_ok("-c", code) + + def test_pending_submodule_is_still_tracked(self): + """`lazy from` a submodule that is not loaded must stay tracked.""" + code = textwrap.dedent(""" + import sys + lazy from test.test_lazy_import.data.pkg import b + assert "test.test_lazy_import.data.pkg.b" in sys.lazy_modules, ( + f"expected 'pkg.b' tracked, got {sys.lazy_modules}" + ) + import test.test_lazy_import.data.pkg + assert "test.test_lazy_import.data.pkg.b" not in sys.modules, ( + "loading the package must not load the submodule" + ) + assert "test.test_lazy_import.data.pkg.b" in sys.lazy_modules, ( + f"loading the package must not untrack the submodule, " + f"got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + @support.requires_subprocess() class CommandLineAndEnvVarTests(unittest.TestCase): diff --git a/Lib/test/test_lazy_import/data/init_fails.py b/Lib/test/test_lazy_import/data/init_fails.py new file mode 100644 index 000000000000000..16e7ad363b0eba6 --- /dev/null +++ b/Lib/test/test_lazy_import/data/init_fails.py @@ -0,0 +1,4 @@ +# Imported by test_initializing_module_is_still_tracked. The module it imports +# lazily imports this one back while this one is still initializing. +import test.test_lazy_import.data.lazy_on_init_fails +raise ValueError("initialization failed") diff --git a/Lib/test/test_lazy_import/data/lazy_on_init_fails.py b/Lib/test/test_lazy_import/data/lazy_on_init_fails.py new file mode 100644 index 000000000000000..7203fcbb4aaa452 --- /dev/null +++ b/Lib/test/test_lazy_import/data/lazy_on_init_fails.py @@ -0,0 +1 @@ +lazy import test.test_lazy_import.data.init_fails diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-16-15-02-11.gh-issue-155695.Kf3xQa.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-16-15-02-11.gh-issue-155695.Kf3xQa.rst new file mode 100644 index 000000000000000..f0e8c7c849be220 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-16-15-02-11.gh-issue-155695.Kf3xQa.rst @@ -0,0 +1,3 @@ +Resolved names are now removed from :data:`sys.lazy_modules` more +consistently: a lazy import of an already loaded module is no longer recorded, +and reifying ``lazy from pkg import attr`` now discards the ``pkg.attr`` entry. diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index e4a949a5456cfd2..69b3e1663d0786a 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -452,6 +452,14 @@ lazy_import_resolve_impl(PyThreadState *tstate, PyObject *lazy_import, lazy_import_add_exception_cause(tstate, lz); } assert(obj == NULL || !PyLazyImport_CheckExact(obj)); + if (obj != NULL) { + PyObject *name = lazy_import_name(lz); + if (name == NULL || + _PyImport_DiscardLazyModule(tstate->interp, name) < 0) { + Py_CLEAR(obj); + } + Py_XDECREF(name); + } if (resolving != NULL) { // A failed set resize can leave the placeholder inserted. Removing by // identity also permits greenlets to finish in a different order. diff --git a/Python/import.c b/Python/import.c index a3b07facc1afa7f..f2aeb18fc9edd7b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -286,6 +286,12 @@ _PyImport_ClearLazyModules(PyInterpreterState *interp) Py_CLEAR(LAZY_PENDING_SUBMODULES(interp)); } +int +_PyImport_DiscardLazyModule(PyInterpreterState *interp, PyObject *name) +{ + return PySet_Discard(LAZY_MODULES(interp), name); +} + static PyObject * get_importtime_name(PyObject *name) { @@ -4125,6 +4131,47 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } +// Add *name* to `sys.lazy_modules`. +// Skip the registration if *name* is already imported, and fully initialized. +// If *existing_mod_p* is not NULL, set it to the already imported object (or +// NULL if not found), for use by register_lazy_on_parent. +static int +lazy_modules_add(PyThreadState *tstate, PyObject *name, + PyObject **existing_mod_p) +{ + PyObject *mod = import_get_module(tstate, name); + if (mod == NULL && PyErr_Occurred()) { + return -1; + } + int loaded = mod != NULL && mod != Py_None; + if (loaded && PyModule_Check(mod)) { + PyObject *spec; + if (PyDict_GetItemRef(_PyModule_GetDict(mod), &_Py_ID(__spec__), &spec) < 0) { + Py_DECREF(mod); + return -1; + } + loaded = spec == NULL || spec == Py_None; + if (!loaded) { + // Read the usual flag without callbacks or dictionary allocation. + // Unsupported spec representations stay conservatively tracked. + PyObject *initializing = NULL; + if ((Py_TYPE(spec)->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_TryGetInstanceAttribute(spec, &_Py_ID(_initializing), &initializing)) { + loaded = initializing == NULL || initializing == Py_False; + } + Py_XDECREF(initializing); + } + Py_XDECREF(spec); + } + if (existing_mod_p != NULL) { + *existing_mod_p = mod; + } + else { + Py_XDECREF(mod); + } + return loaded ? 0 : PySet_Add(LAZY_MODULES(tstate->interp), name); +} + // Ensure a set of pending submodule names exists for the parent. // Returns a new reference. static PyObject * @@ -4186,22 +4233,30 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name) } static int -register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, - PyObject *from) -{ +register_from_lazy_on_parent(PyThreadState *tstate, PyObject *existing_module, + PyObject *abs_name, PyObject *from) +{ + // IMPORT_FROM returns stored attributes directly. Their imports are + // already resolved or tracked by their own placeholders, so skip the alias. + int rc = 0; + if (existing_module != NULL && PyModule_Check(existing_module)) { + rc = PyDict_Contains(_PyModule_GetDict(existing_module), from); + if (rc != 0) { + return rc < 0 ? -1 : 0; + } + } + PyObject *fromname = PyUnicode_FromFormat("%U.%U", abs_name, from); if (fromname == NULL) { return -1; } - // Add the module name to sys.lazy_modules set (PEP 810). - PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - int res = PySet_Add(lazy_modules, fromname); - if (res == 0) { - res = register_lazy_on_parent(tstate, fromname); + rc = lazy_modules_add(tstate, fromname, NULL); + if (rc == 0) { + rc = register_lazy_on_parent(tstate, fromname); } Py_DECREF(fromname); - return res; + return rc; } PyObject * @@ -4288,6 +4343,8 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, } PyObject *res = NULL; + PyObject *existing_module = NULL; + if (fromlist != NULL && PyUnicode_Check(fromlist)) { fromlist = PyTuple_Pack(1, fromlist); if (fromlist == NULL) { @@ -4352,15 +4409,13 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, goto done; } - // Add the module name to sys.lazy_modules set (PEP 810). - PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - if (PySet_Add(lazy_modules, abs_name) < 0) { + if (lazy_modules_add(tstate, abs_name, &existing_module) < 0) { goto error; } if (PyTuple_Check(fromlist) && PyTuple_GET_SIZE(fromlist)) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { - if (register_from_lazy_on_parent(tstate, abs_name, + if (register_from_lazy_on_parent(tstate, existing_module, abs_name, PyTuple_GET_ITEM(fromlist, i)) < 0) { goto error; @@ -4377,6 +4432,7 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, done: Py_XDECREF(fromlist); Py_DECREF(abs_name); + Py_XDECREF(existing_module); return res; } _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
