https://github.com/python/cpython/commit/aa0e4109a8681cbe6fb65e4072b0058e02adaef9
commit: aa0e4109a8681cbe6fb65e4072b0058e02adaef9
branch: main
author: Lysandros Nikolaou <[email protected]>
committer: lysnikolaou <[email protected]>
date: 2026-10-25T10:08:26-07:00
summary:
gh-153568: Use a managed dict for AST nodes (#160616)
AST nodes had an explicit `__dict__` slot, so every node built by
`ast.parse` allocated its own dict. With `Py_TPFLAGS_MANAGED_DICT`,
node attributes are stored as inline values with shared keys instead.
`ast.parse` is ~15.8% faster, walking trees is ~15.9% faster, and
the memory used by the AST trees is reduced by ~51.6%.
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-23-45-00.gh-issue-153568.astinl.rst
A
Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst
M Lib/test/test_ast/test_ast.py
M Lib/test/test_sys.py
M Parser/asdl_c.py
M Python/Python-ast.c
diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py
index b1483ad6bf2d401..08d4cfb9ae6c8a2 100644
--- a/Lib/test/test_ast/test_ast.py
+++ b/Lib/test/test_ast/test_ast.py
@@ -103,6 +103,13 @@ def test_AST_objects(self):
# "ast.AST constructor takes 0 positional arguments"
ast.AST(2)
+ @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi")
+ def test_nodes_have_inline_values(self):
+ tree = ast.parse("x = f(y, z=1)")
+ for node in ast.walk(tree):
+ with self.subTest(node=type(node).__name__):
+ self.assertTrue(_testinternalcapi.has_inline_values(node))
+
def test_AST_fields_NULL_check(self):
# See: https://github.com/python/cpython/issues/126105
old_value = ast.AST._fields
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index 3de66a5508e69a4..201c209d8214150 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -2042,7 +2042,7 @@ def test_pythontypes(self):
check = self.check_sizeof
# _ast.AST
import _ast
- check(_ast.Module(), size('3P'))
+ check(_ast.Module(), size('2P'))
try:
raise TypeError
except TypeError as e:
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-23-45-00.gh-issue-153568.astinl.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-23-45-00.gh-issue-153568.astinl.rst
new file mode 100644
index 000000000000000..a043d20b7a3408b
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-23-45-00.gh-issue-153568.astinl.rst
@@ -0,0 +1,2 @@
+Speed up :func:`ast.parse` and reduce the memory used by AST nodes by storing
+node attributes as inline values instead of a separate per-node dictionary.
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst
new file mode 100644
index 000000000000000..98628f6a74208f7
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst
@@ -0,0 +1,3 @@
+Speed up :func:`ast.parse` by ~15.8% and reduce the memory used by ~51.6% by
+storing node attributes as inline values instead of in a per-node
+dictionary.
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 2b83e97353bbf1f..edbb749f5ee231b 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -913,7 +913,6 @@ def visitModule(self, mod):
typedef struct {
PyObject_HEAD
- PyObject *dict;
} AST_object;
static void
@@ -923,7 +922,7 @@ def visitModule(self, mod):
/* bpo-31095: UnTrack is needed before calling any callbacks */
PyTypeObject *tp = Py_TYPE(self);
PyObject_GC_UnTrack(self);
- Py_CLEAR(self->dict);
+ PyObject_ClearManagedDict(op);
freefunc free_func = PyType_GetSlot(tp, Py_tp_free);
assert(free_func != NULL);
free_func(self);
@@ -935,15 +934,13 @@ def visitModule(self, mod):
{
AST_object *self = (AST_object*)op;
Py_VISIT(Py_TYPE(self));
- Py_VISIT(self->dict);
- return 0;
+ return PyObject_VisitManagedDict(op, visit, arg);
}
static int
ast_clear(PyObject *op)
{
- AST_object *self = (AST_object*)op;
- Py_CLEAR(self->dict);
+ PyObject_ClearManagedDict(op);
return 0;
}
@@ -1419,11 +1416,6 @@ def visitModule(self, mod):
return result;
}
-static PyMemberDef ast_type_members[] = {
- {"__dictoffset__", Py_T_PYSSIZET, offsetof(AST_object, dict), Py_READONLY},
- {NULL} /* Sentinel */
-};
-
static PyMethodDef ast_type_methods[] = {
{"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
{"__replace__", _PyCFunction_CAST(ast_type_replace), METH_VARARGS |
METH_KEYWORDS,
@@ -1668,7 +1660,6 @@ def visitModule(self, mod):
{Py_tp_setattro, PyObject_GenericSetAttr},
{Py_tp_traverse, ast_traverse},
{Py_tp_clear, ast_clear},
- {Py_tp_members, ast_type_members},
{Py_tp_methods, ast_type_methods},
{Py_tp_getset, ast_type_getsets},
{Py_tp_init, ast_type_init},
@@ -1682,7 +1673,8 @@ def visitModule(self, mod):
"ast.AST",
sizeof(AST_object),
0,
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC |
+ Py_TPFLAGS_MANAGED_DICT,
AST_type_slots
};
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 0d89f7e87f1fa2d..2c020e6c2070fc4 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -1414,7 +1414,6 @@ add_ast_annotations(struct ast_state *state)
typedef struct {
PyObject_HEAD
- PyObject *dict;
} AST_object;
static void
@@ -1424,7 +1423,7 @@ ast_dealloc(PyObject *op)
/* bpo-31095: UnTrack is needed before calling any callbacks */
PyTypeObject *tp = Py_TYPE(self);
PyObject_GC_UnTrack(self);
- Py_CLEAR(self->dict);
+ PyObject_ClearManagedDict(op);
freefunc free_func = PyType_GetSlot(tp, Py_tp_free);
assert(free_func != NULL);
free_func(self);
@@ -1436,15 +1435,13 @@ ast_traverse(PyObject *op, visitproc visit, void *arg)
{
AST_object *self = (AST_object*)op;
Py_VISIT(Py_TYPE(self));
- Py_VISIT(self->dict);
- return 0;
+ return PyObject_VisitManagedDict(op, visit, arg);
}
static int
ast_clear(PyObject *op)
{
- AST_object *self = (AST_object*)op;
- Py_CLEAR(self->dict);
+ PyObject_ClearManagedDict(op);
return 0;
}
@@ -1920,11 +1917,6 @@ ast_type_replace(PyObject *self, PyObject *args,
PyObject *kwargs)
return result;
}
-static PyMemberDef ast_type_members[] = {
- {"__dictoffset__", Py_T_PYSSIZET, offsetof(AST_object, dict), Py_READONLY},
- {NULL} /* Sentinel */
-};
-
static PyMethodDef ast_type_methods[] = {
{"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
{"__replace__", _PyCFunction_CAST(ast_type_replace), METH_VARARGS |
METH_KEYWORDS,
@@ -2169,7 +2161,6 @@ static PyType_Slot AST_type_slots[] = {
{Py_tp_setattro, PyObject_GenericSetAttr},
{Py_tp_traverse, ast_traverse},
{Py_tp_clear, ast_clear},
- {Py_tp_members, ast_type_members},
{Py_tp_methods, ast_type_methods},
{Py_tp_getset, ast_type_getsets},
{Py_tp_init, ast_type_init},
@@ -2183,7 +2174,8 @@ static PyType_Spec AST_type_spec = {
"ast.AST",
sizeof(AST_object),
0,
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC |
+ Py_TPFLAGS_MANAGED_DICT,
AST_type_slots
};
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
