https://github.com/python/cpython/commit/bdedc4a20e09ea2e6af64bd0430724e44056207e commit: bdedc4a20e09ea2e6af64bd0430724e44056207e branch: main author: Brian Schubert <[email protected]> committer: JelleZijlstra <[email protected]> date: 2026-05-02T09:50:06-07:00 summary:
gh-116021: Deprecate support for instantiating abstract AST nodes (#137865) Co-authored-by: Bénédikt Tran <[email protected]> Co-authored-by: Jelle Zijlstra <[email protected]> files: A Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst M Doc/deprecations/pending-removal-in-3.20.rst M Doc/library/ast.rst M Doc/whatsnew/3.15.rst M Include/internal/pycore_ast_state.h M Lib/test/test_ast/test_ast.py M Lib/test/test_sys.py M Parser/asdl_c.py M Python/Python-ast.c diff --git a/Doc/deprecations/pending-removal-in-3.20.rst b/Doc/deprecations/pending-removal-in-3.20.rst index 176e8f3f9f601c..12d7acf5ce05b4 100644 --- a/Doc/deprecations/pending-removal-in-3.20.rst +++ b/Doc/deprecations/pending-removal-in-3.20.rst @@ -38,3 +38,8 @@ Pending removal in Python 3.20 - :mod:`zlib` (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) + +* :mod:`ast`: + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 3c6e8745474316..18df18d0c05421 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -42,7 +42,7 @@ Node classes .. class:: AST - This is the base of all AST node classes. The actual node classes are + This is the abstract base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced :ref:`above <abstract-grammar>`. They are defined in the :mod:`!_ast` C module and re-exported in :mod:`!ast`. @@ -168,6 +168,15 @@ Node classes arguments that were set as attributes of the AST node, even if they did not match any of the fields of the AST node. These cases now raise a :exc:`TypeError`. +.. deprecated-removed:: next 3.20 + + In the :ref:`grammar above <abstract-grammar>`, the AST node classes that + correspond to production rules with variants (aka "sums") are abstract + classes. Previous versions of Python allowed for the creation of direct + instances of these abstract node classes. This behavior is deprecated and + will be removed in Python 3.20. + + .. note:: The descriptions of the specific node classes displayed here were initially adapted from the fantastic `Green Tree diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index b63e7a4790e9af..78e464f2a5a6d8 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1846,6 +1846,13 @@ Deprecated New deprecations ---------------- +* :mod:`ast` + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. + + (Contributed by Brian Schubert in :gh:`116021`.) + * :mod:`base64`: * Accepting the ``+`` and ``/`` characters with an alternative alphabet in diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index 1caf200ee34b2a..32c12fb5875e8e 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -161,6 +161,7 @@ struct ast_state { PyObject *__module__; PyObject *_attributes; PyObject *_fields; + PyObject *abstract_types; PyObject *alias_type; PyObject *annotation; PyObject *arg; diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index a7d5a51a2aa4d5..fcc6c93eb86981 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1,4 +1,5 @@ import _ast_unparse +import _ast import ast import builtins import contextlib @@ -85,7 +86,9 @@ def _assertTrueorder(self, ast_node, parent_pos): self.assertEqual(ast_node._fields, ast_node.__match_args__) def test_AST_objects(self): - x = ast.AST() + # Directly instantiating abstract node class AST is allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.AST() self.assertEqual(x._fields, ()) x.foobar = 42 self.assertEqual(x.foobar, 42) @@ -94,7 +97,7 @@ def test_AST_objects(self): with self.assertRaises(AttributeError): x.vararg - with self.assertRaises(TypeError): + with self.assertRaises(TypeError), self.assertWarns(DeprecationWarning): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) @@ -110,15 +113,21 @@ def cleanup(): msg = "type object 'ast.AST' has no attribute '_fields'" # Both examples used to crash: - with self.assertRaisesRegex(AttributeError, msg): + with ( + self.assertRaisesRegex(AttributeError, msg), + self.assertWarns(DeprecationWarning), + ): ast.AST(arg1=123) - with self.assertRaisesRegex(AttributeError, msg): + with ( + self.assertRaisesRegex(AttributeError, msg), + self.assertWarns(DeprecationWarning), + ): ast.AST() - def test_AST_garbage_collection(self): + def test_node_garbage_collection(self): class X: pass - a = ast.AST() + a = ast.Module() a.x = X() a.x.a = a ref = weakref.ref(a.x) @@ -439,7 +448,15 @@ def _construct_ast_class(self, cls): elif typ is object: kwargs[name] = b'capybara' elif isinstance(typ, type) and issubclass(typ, ast.AST): - kwargs[name] = self._construct_ast_class(typ) + if _ast._is_abstract(typ): + # Use an arbitrary concrete subclass + concrete = next((sub for sub in typ.__subclasses__() + if not _ast._is_abstract(sub)), None) + msg = f"abstract node class {typ} has no concrete subclasses" + self.assertIsNotNone(concrete, msg) + else: + concrete = typ + kwargs[name] = self._construct_ast_class(concrete) return cls(**kwargs) def test_arguments(self): @@ -578,6 +595,10 @@ def test_nodeclasses(self): with self.assertRaisesRegex(TypeError, re.escape(msg)): ast.BinOp(1, 2, 3, foobarbaz=42) + # Directly instantiating abstract node types is allowed (but deprecated) + self.assertWarns(DeprecationWarning, ast.stmt) + self.assertWarns(DeprecationWarning, ast.expr_context) + def test_no_fields(self): # this used to fail because Sub._fields was None x = ast.Sub() @@ -585,7 +606,10 @@ def test_no_fields(self): def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) - m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) + with self.assertWarns(DeprecationWarning): + # Creating instances of ast.expr is deprecated + e = ast.expr(**pos) + m = ast.Module([ast.Expr(e, **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "<test>", "exec") self.assertIn("but got expr()", str(cm.exception)) @@ -1107,14 +1131,19 @@ class CopyTests(unittest.TestCase): def iter_ast_classes(): """Iterate over the (native) subclasses of ast.AST recursively. - This excludes the special class ast.Index since its constructor - returns an integer. + This excludes: + * abstract AST nodes + * the special class ast.Index, since its constructor returns + an integer. """ def do(cls): if cls.__module__ != 'ast': return if cls is ast.Index: return + # Don't attempt to create instances of abstract AST nodes + if _ast._is_abstract(cls): + return yield cls for sub in cls.__subclasses__(): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 34a773ee9376cd..3002fa528eab17 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1908,7 +1908,7 @@ def test_pythontypes(self): check = self.check_sizeof # _ast.AST import _ast - check(_ast.AST(), size('P')) + check(_ast.Module(), size('3P')) try: raise TypeError except TypeError as e: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst new file mode 100644 index 00000000000000..967d8faaef3422 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst @@ -0,0 +1,2 @@ +Support for creating instances of abstract AST nodes from the :mod:`ast` module +is deprecated and scheduled for removal in Python 3.20. Patch by Brian Schubert. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index df4454cf948693..ed1d2e08df8f40 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -945,6 +945,19 @@ def visitModule(self, mod): return -1; } + int contains = PySet_Contains(state->abstract_types, (PyObject *)Py_TYPE(self)); + if (contains == -1) { + return -1; + } + else if (contains == 1) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Instantiating abstract AST node class %T is deprecated. " + "This will become an error in Python 3.20", self) < 0) { + return -1; + } + } + Py_ssize_t i, numfields = 0; int res = -1; PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL; @@ -1777,6 +1790,13 @@ def visitModule(self, mod): if (!state->AST_type) { return -1; } + state->abstract_types = PySet_New(NULL); + if (!state->abstract_types) { + return -1; + } + if (PySet_Add(state->abstract_types, state->AST_type) < 0) { + return -1; + } if (add_ast_fields(state) < 0) { return -1; } @@ -1818,6 +1838,7 @@ def visitSum(self, sum, name): (name, name, len(sum.attributes)), 1) else: self.emit("if (add_attributes(state, state->%s_type, NULL, 0) < 0) return -1;" % name, 1) + self.emit("if (PySet_Add(state->abstract_types, state->%s_type) < 0) return -1;" % name, 1) self.emit_defaults(name, sum.attributes, 1) simple = is_simple(sum) for t in sum.types: @@ -1850,6 +1871,30 @@ def emit_defaults(self, name, fields, depth): class ASTModuleVisitor(PickleVisitor): def visitModule(self, mod): + self.emit(""" +/* Helper for checking if a node class is abstract in the tests. */ +static PyObject * +ast_is_abstract(PyObject *Py_UNUSED(module), PyObject *cls) { + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + int contains = PySet_Contains(state->abstract_types, cls); + if (contains == -1) { + return NULL; + } + else if (contains == 1) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static struct PyMethodDef astmodule_methods[] = { + {"_is_abstract", ast_is_abstract, METH_O, NULL}, + {NULL} /* Sentinel */ +}; +""".strip(), 0, reflow=False) + self.emit("", 0) self.emit("static int", 0) self.emit("astmodule_exec(PyObject *m)", 0) self.emit("{", 0) @@ -1891,7 +1936,8 @@ def visitModule(self, mod): .m_name = "_ast", // The _ast module uses a per-interpreter state (PyInterpreterState.ast) .m_size = 0, - .m_slots = astmodule_slots, + .m_methods = astmodule_methods, + .m_slots = astmodule_slots }; PyMODINIT_FUNC @@ -2180,6 +2226,7 @@ def generate_module_def(mod, metadata, f, internal_h): "%s_type" % type for type in metadata.types ) + module_state.add("abstract_types") state_strings = sorted(state_strings) module_state = sorted(module_state) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 6bcf57bdd6b4f4..5f3d9c4b17410f 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -178,6 +178,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->__module__); Py_CLEAR(state->_attributes); Py_CLEAR(state->_fields); + Py_CLEAR(state->abstract_types); Py_CLEAR(state->alias_type); Py_CLEAR(state->annotation); Py_CLEAR(state->arg); @@ -5269,6 +5270,19 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) return -1; } + int contains = PySet_Contains(state->abstract_types, (PyObject *)Py_TYPE(self)); + if (contains == -1) { + return -1; + } + else if (contains == 1) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Instantiating abstract AST node class %T is deprecated. " + "This will become an error in Python 3.20", self) < 0) { + return -1; + } + } + Py_ssize_t i, numfields = 0; int res = -1; PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL; @@ -6100,6 +6114,13 @@ init_types(void *arg) if (!state->AST_type) { return -1; } + state->abstract_types = PySet_New(NULL); + if (!state->abstract_types) { + return -1; + } + if (PySet_Add(state->abstract_types, state->AST_type) < 0) { + return -1; + } if (add_ast_fields(state) < 0) { return -1; } @@ -6110,6 +6131,7 @@ init_types(void *arg) " | FunctionType(expr* argtypes, expr returns)"); if (!state->mod_type) return -1; if (add_attributes(state, state->mod_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->mod_type) < 0) return -1; state->Module_type = make_type(state, "Module", state->mod_type, Module_fields, 2, "Module(stmt* body, type_ignore* type_ignores)"); @@ -6159,6 +6181,7 @@ init_types(void *arg) if (!state->stmt_type) return -1; if (add_attributes(state, state->stmt_type, stmt_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->stmt_type) < 0) return -1; if (PyObject_SetAttr(state->stmt_type, state->end_lineno, Py_None) == -1) return -1; if (PyObject_SetAttr(state->stmt_type, state->end_col_offset, Py_None) == @@ -6348,6 +6371,7 @@ init_types(void *arg) if (!state->expr_type) return -1; if (add_attributes(state, state->expr_type, expr_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->expr_type) < 0) return -1; if (PyObject_SetAttr(state->expr_type, state->end_lineno, Py_None) == -1) return -1; if (PyObject_SetAttr(state->expr_type, state->end_col_offset, Py_None) == @@ -6494,6 +6518,8 @@ init_types(void *arg) "expr_context = Load | Store | Del"); if (!state->expr_context_type) return -1; if (add_attributes(state, state->expr_context_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->expr_context_type) < 0) return + -1; state->Load_type = make_type(state, "Load", state->expr_context_type, NULL, 0, "Load"); @@ -6518,6 +6544,7 @@ init_types(void *arg) "boolop = And | Or"); if (!state->boolop_type) return -1; if (add_attributes(state, state->boolop_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->boolop_type) < 0) return -1; state->And_type = make_type(state, "And", state->boolop_type, NULL, 0, "And"); if (!state->And_type) return -1; @@ -6535,6 +6562,7 @@ init_types(void *arg) "operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv"); if (!state->operator_type) return -1; if (add_attributes(state, state->operator_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->operator_type) < 0) return -1; state->Add_type = make_type(state, "Add", state->operator_type, NULL, 0, "Add"); if (!state->Add_type) return -1; @@ -6629,6 +6657,7 @@ init_types(void *arg) "unaryop = Invert | Not | UAdd | USub"); if (!state->unaryop_type) return -1; if (add_attributes(state, state->unaryop_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->unaryop_type) < 0) return -1; state->Invert_type = make_type(state, "Invert", state->unaryop_type, NULL, 0, "Invert"); @@ -6659,6 +6688,7 @@ init_types(void *arg) "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn"); if (!state->cmpop_type) return -1; if (add_attributes(state, state->cmpop_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->cmpop_type) < 0) return -1; state->Eq_type = make_type(state, "Eq", state->cmpop_type, NULL, 0, "Eq"); if (!state->Eq_type) return -1; @@ -6732,6 +6762,8 @@ init_types(void *arg) if (!state->excepthandler_type) return -1; if (add_attributes(state, state->excepthandler_type, excepthandler_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->excepthandler_type) < 0) return + -1; if (PyObject_SetAttr(state->excepthandler_type, state->end_lineno, Py_None) == -1) return -1; @@ -6822,6 +6854,7 @@ init_types(void *arg) if (!state->pattern_type) return -1; if (add_attributes(state, state->pattern_type, pattern_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->pattern_type) < 0) return -1; state->MatchValue_type = make_type(state, "MatchValue", state->pattern_type, MatchValue_fields, 1, @@ -6872,6 +6905,8 @@ init_types(void *arg) "type_ignore = TypeIgnore(int lineno, string tag)"); if (!state->type_ignore_type) return -1; if (add_attributes(state, state->type_ignore_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->type_ignore_type) < 0) return + -1; state->TypeIgnore_type = make_type(state, "TypeIgnore", state->type_ignore_type, TypeIgnore_fields, 2, @@ -6885,6 +6920,7 @@ init_types(void *arg) if (!state->type_param_type) return -1; if (add_attributes(state, state->type_param_type, type_param_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->type_param_type) < 0) return -1; state->TypeVar_type = make_type(state, "TypeVar", state->type_param_type, TypeVar_fields, 3, "TypeVar(identifier name, expr? bound, expr? default_value)"); @@ -17956,6 +17992,28 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, } +/* Helper for checking if a node class is abstract in the tests. */ +static PyObject * +ast_is_abstract(PyObject *Py_UNUSED(module), PyObject *cls) { + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + int contains = PySet_Contains(state->abstract_types, cls); + if (contains == -1) { + return NULL; + } + else if (contains == 1) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static struct PyMethodDef astmodule_methods[] = { + {"_is_abstract", ast_is_abstract, METH_O, NULL}, + {NULL} /* Sentinel */ +}; + static int astmodule_exec(PyObject *m) { @@ -18382,7 +18440,8 @@ static struct PyModuleDef _astmodule = { .m_name = "_ast", // The _ast module uses a per-interpreter state (PyInterpreterState.ast) .m_size = 0, - .m_slots = astmodule_slots, + .m_methods = astmodule_methods, + .m_slots = astmodule_slots }; PyMODINIT_FUNC _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
