https://github.com/python/cpython/commit/07b8d3117fdbc4e5be55aab0be428c278ec84e12
commit: 07b8d3117fdbc4e5be55aab0be428c278ec84e12
branch: main
author: Jelle Zijlstra <[email protected]>
committer: JelleZijlstra <[email protected]>
date: 2025-04-10T21:13:26-07:00
summary:
gh-132261: Store annotations at hidden internal keys in the class dict (#132345)
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2025-04-09-21-51-37.gh-issue-132261.gL8thm.rst
M Doc/library/annotationlib.rst
M Include/internal/pycore_global_objects_fini_generated.h
M Include/internal/pycore_global_strings.h
M Include/internal/pycore_magic_number.h
M Include/internal/pycore_runtime_init_generated.h
M Include/internal/pycore_unicodeobject_generated.h
M Lib/annotationlib.py
M Lib/pydoc.py
M Lib/test/test_ast/test_ast.py
M Lib/test/test_pydoc/test_pydoc.py
M Lib/test/test_type_annotations.py
M Lib/test/test_typing.py
M Lib/typing.py
M Objects/typeobject.c
M Python/codegen.c
diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst
index e07081e3c5dd7a..140e1aa12e2938 100644
--- a/Doc/library/annotationlib.rst
+++ b/Doc/library/annotationlib.rst
@@ -303,12 +303,12 @@ Functions
.. function:: get_annotate_function(obj)
Retrieve the :term:`annotate function` for *obj*. Return :const:`!None`
- if *obj* does not have an annotate function.
+ if *obj* does not have an annotate function. *obj* may be a class, function,
+ module, or a namespace dictionary for a class. The last case is useful
during
+ class creation, e.g. in the ``__new__`` method of a metaclass.
This is usually equivalent to accessing the :attr:`~object.__annotate__`
- attribute of *obj*, but direct access to the attribute may return the wrong
- object in certain situations involving metaclasses. This function should be
- used instead of accessing the attribute directly.
+ attribute of *obj*, but access through this public function is preferred.
.. versionadded:: 3.14
diff --git a/Include/internal/pycore_global_objects_fini_generated.h
b/Include/internal/pycore_global_objects_fini_generated.h
index 410a3734f1a607..5485d0bd64f3f1 100644
--- a/Include/internal/pycore_global_objects_fini_generated.h
+++ b/Include/internal/pycore_global_objects_fini_generated.h
@@ -587,7 +587,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__and__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__anext__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate__));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotate_func__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations__));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__annotations_cache__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__args__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bases__));
diff --git a/Include/internal/pycore_global_strings.h
b/Include/internal/pycore_global_strings.h
index cadbc01b01de26..3ce192511e3879 100644
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -78,7 +78,9 @@ struct _Py_global_strings {
STRUCT_FOR_ID(__and__)
STRUCT_FOR_ID(__anext__)
STRUCT_FOR_ID(__annotate__)
+ STRUCT_FOR_ID(__annotate_func__)
STRUCT_FOR_ID(__annotations__)
+ STRUCT_FOR_ID(__annotations_cache__)
STRUCT_FOR_ID(__args__)
STRUCT_FOR_ID(__await__)
STRUCT_FOR_ID(__bases__)
diff --git a/Include/internal/pycore_magic_number.h
b/Include/internal/pycore_magic_number.h
index 3fa2b714cb6f23..f75b05893affc1 100644
--- a/Include/internal/pycore_magic_number.h
+++ b/Include/internal/pycore_magic_number.h
@@ -274,6 +274,7 @@ Known values:
Python 3.14a6 3619 (Renumber RESUME opcode from 149 to 128)
Python 3.14a6 3620 (Optimize bytecode for all/any/tuple called on a genexp)
Python 3.14a7 3621 (Optimize LOAD_FAST opcodes into LOAD_FAST_BORROW)
+ Python 3.14a7 3622 (Store annotations in different class dict keys)
Python 3.15 will start with 3650
@@ -286,7 +287,7 @@ PC/launcher.c must also be updated.
*/
-#define PYC_MAGIC_NUMBER 3621
+#define PYC_MAGIC_NUMBER 3622
/* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
(little-endian) and then appending b'\r\n'. */
#define PYC_MAGIC_NUMBER_TOKEN \
diff --git a/Include/internal/pycore_runtime_init_generated.h
b/Include/internal/pycore_runtime_init_generated.h
index 07a74dd26cd11f..5c95d0feddecba 100644
--- a/Include/internal/pycore_runtime_init_generated.h
+++ b/Include/internal/pycore_runtime_init_generated.h
@@ -585,7 +585,9 @@ extern "C" {
INIT_ID(__and__), \
INIT_ID(__anext__), \
INIT_ID(__annotate__), \
+ INIT_ID(__annotate_func__), \
INIT_ID(__annotations__), \
+ INIT_ID(__annotations_cache__), \
INIT_ID(__args__), \
INIT_ID(__await__), \
INIT_ID(__bases__), \
diff --git a/Include/internal/pycore_unicodeobject_generated.h
b/Include/internal/pycore_unicodeobject_generated.h
index 1e1e32bbd42eed..a1fc9736d66618 100644
--- a/Include/internal/pycore_unicodeobject_generated.h
+++ b/Include/internal/pycore_unicodeobject_generated.h
@@ -100,10 +100,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
+ string = &_Py_ID(__annotate_func__);
+ _PyUnicode_InternStatic(interp, &string);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(__annotations__);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
assert(PyUnicode_GET_LENGTH(string) != 1);
+ string = &_Py_ID(__annotations_cache__);
+ _PyUnicode_InternStatic(interp, &string);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ assert(PyUnicode_GET_LENGTH(string) != 1);
string = &_Py_ID(__args__);
_PyUnicode_InternStatic(interp, &string);
assert(_PyUnicode_CheckConsistency(string, 1));
diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py
index d6243c8863610e..237b3470b831fd 100644
--- a/Lib/annotationlib.py
+++ b/Lib/annotationlib.py
@@ -619,14 +619,6 @@ def call_annotate_function(annotate, format, *,
owner=None, _is_evaluate=False):
raise ValueError(f"Invalid format: {format!r}")
-# We use the descriptors from builtins.type instead of accessing
-# .__annotations__ and .__annotate__ directly on class objects, because
-# otherwise we could get wrong results in some cases involving metaclasses.
-# See PEP 749.
-_BASE_GET_ANNOTATE = type.__dict__["__annotate__"].__get__
-_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__
-
-
def get_annotate_function(obj):
"""Get the __annotate__ function for an object.
@@ -635,12 +627,11 @@ def get_annotate_function(obj):
Returns the __annotate__ function or None.
"""
- if isinstance(obj, type):
+ if isinstance(obj, dict):
try:
- return _BASE_GET_ANNOTATE(obj)
- except AttributeError:
- # AttributeError is raised for static types.
- return None
+ return obj["__annotate__"]
+ except KeyError:
+ return obj.get("__annotate_func__", None)
return getattr(obj, "__annotate__", None)
@@ -833,7 +824,7 @@ def _get_and_call_annotate(obj, format):
def _get_dunder_annotations(obj):
if isinstance(obj, type):
try:
- ann = _BASE_GET_ANNOTATIONS(obj)
+ ann = obj.__annotations__
except AttributeError:
# For static types, the descriptor raises AttributeError.
return {}
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
index 1839b88fec28b1..169194b99cb826 100644
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -330,7 +330,8 @@ def visiblename(name, all=None, obj=None):
'__date__', '__doc__', '__file__', '__spec__',
'__loader__', '__module__', '__name__', '__package__',
'__path__', '__qualname__', '__slots__', '__version__',
- '__static_attributes__', '__firstlineno__'}:
+ '__static_attributes__', '__firstlineno__',
+ '__annotate_func__', '__annotations_cache__'}:
return 0
# Private names are hidden, but special names are displayed.
if name.startswith('__') and name.endswith('__'): return 1
diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py
index 090544726c31a4..dd459487afef1c 100644
--- a/Lib/test/test_ast/test_ast.py
+++ b/Lib/test/test_ast/test_ast.py
@@ -298,7 +298,7 @@ def test_arguments(self):
x = ast.arguments()
self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg',
'kwonlyargs',
'kw_defaults', 'kwarg', 'defaults'))
- self.assertEqual(x.__annotations__, {
+ self.assertEqual(ast.arguments.__annotations__, {
'posonlyargs': list[ast.arg],
'args': list[ast.arg],
'vararg': ast.arg | None,
diff --git a/Lib/test/test_pydoc/test_pydoc.py
b/Lib/test/test_pydoc/test_pydoc.py
index 2b1a4484c680fc..8cb253f67ea2eb 100644
--- a/Lib/test/test_pydoc/test_pydoc.py
+++ b/Lib/test/test_pydoc/test_pydoc.py
@@ -78,11 +78,6 @@ class A(builtins.object)
| __weakref__%s
class B(builtins.object)
- | Methods defined here:
- |
- | __annotate__(format, /)
- |
- | ----------------------------------------------------------------------
| Data descriptors defined here:
|
| __dict__%s
@@ -180,9 +175,6 @@ class A(builtins.object)
list of weak references to the object
class B(builtins.object)
- Methods defined here:
- __annotate__(format, /)
- ----------------------------------------------------------------------
Data descriptors defined here:
__dict__
dictionary for instance variables
diff --git a/Lib/test/test_type_annotations.py
b/Lib/test/test_type_annotations.py
index 29e2c7a0cd837e..31df7668db0976 100644
--- a/Lib/test/test_type_annotations.py
+++ b/Lib/test/test_type_annotations.py
@@ -4,32 +4,33 @@
import types
import unittest
from test.support import run_code, check_syntax_error, cpython_only
+from test.test_inspect import inspect_stringized_annotations
class TypeAnnotationTests(unittest.TestCase):
def test_lazy_create_annotations(self):
# type objects lazy create their __annotations__ dict on demand.
- # the annotations dict is stored in type.__dict__.
+ # the annotations dict is stored in type.__dict__ (as
__annotations_cache__).
# a freshly created type shouldn't have an annotations dict yet.
foo = type("Foo", (), {})
for i in range(3):
- self.assertFalse("__annotations__" in foo.__dict__)
+ self.assertFalse("__annotations_cache__" in foo.__dict__)
d = foo.__annotations__
- self.assertTrue("__annotations__" in foo.__dict__)
+ self.assertTrue("__annotations_cache__" in foo.__dict__)
self.assertEqual(foo.__annotations__, d)
- self.assertEqual(foo.__dict__['__annotations__'], d)
+ self.assertEqual(foo.__dict__['__annotations_cache__'], d)
del foo.__annotations__
def test_setting_annotations(self):
foo = type("Foo", (), {})
for i in range(3):
- self.assertFalse("__annotations__" in foo.__dict__)
+ self.assertFalse("__annotations_cache__" in foo.__dict__)
d = {'a': int}
foo.__annotations__ = d
- self.assertTrue("__annotations__" in foo.__dict__)
+ self.assertTrue("__annotations_cache__" in foo.__dict__)
self.assertEqual(foo.__annotations__, d)
- self.assertEqual(foo.__dict__['__annotations__'], d)
+ self.assertEqual(foo.__dict__['__annotations_cache__'], d)
del foo.__annotations__
def test_annotations_getset_raises(self):
@@ -53,9 +54,30 @@ class C:
a:int=3
b:str=4
self.assertEqual(C.__annotations__, {"a": int, "b": str})
- self.assertTrue("__annotations__" in C.__dict__)
+ self.assertTrue("__annotations_cache__" in C.__dict__)
del C.__annotations__
- self.assertFalse("__annotations__" in C.__dict__)
+ self.assertFalse("__annotations_cache__" in C.__dict__)
+
+ def test_pep563_annotations(self):
+ isa = inspect_stringized_annotations
+ self.assertEqual(
+ isa.__annotations__, {"a": "int", "b": "str"},
+ )
+ self.assertEqual(
+ isa.MyClass.__annotations__, {"a": "int", "b": "str"},
+ )
+
+ def test_explicitly_set_annotations(self):
+ class C:
+ __annotations__ = {"what": int}
+ self.assertEqual(C.__annotations__, {"what": int})
+
+ def test_explicitly_set_annotate(self):
+ class C:
+ __annotate__ = lambda format: {"what": int}
+ self.assertEqual(C.__annotations__, {"what": int})
+ self.assertIsInstance(C.__annotate__, types.FunctionType)
+ self.assertEqual(C.__annotate__(annotationlib.Format.VALUE), {"what":
int})
def test_del_annotations_and_annotate(self):
# gh-132285
diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index edf3cf9d4a3658..9f9e3eb17b9fc9 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -3825,6 +3825,7 @@ def meth(self): pass
acceptable_extra_attrs = {
'_is_protocol', '_is_runtime_protocol', '__parameters__',
'__init__', '__annotations__', '__subclasshook__', '__annotate__',
+ '__annotations_cache__', '__annotate_func__',
}
self.assertLessEqual(vars(NonP).keys(), vars(C).keys() |
acceptable_extra_attrs)
self.assertLessEqual(
diff --git a/Lib/typing.py b/Lib/typing.py
index e67284fc571b1b..e5d14b03a4fc94 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -1784,7 +1784,7 @@ class _TypingEllipsis:
'__init__', '__module__', '__new__', '__slots__',
'__subclasshook__', '__weakref__', '__class_getitem__',
'__match_args__', '__static_attributes__', '__firstlineno__',
- '__annotate__',
+ '__annotate__', '__annotate_func__', '__annotations_cache__',
})
# These special attributes will be not collected as protocol members.
@@ -2875,7 +2875,8 @@ def annotate(format):
'_fields', '_field_defaults',
'_make', '_replace', '_asdict', '_source'})
-_special = frozenset({'__module__', '__name__', '__annotations__',
'__annotate__'})
+_special = frozenset({'__module__', '__name__', '__annotations__',
'__annotate__',
+ '__annotate_func__', '__annotations_cache__'})
class NamedTupleMeta(type):
@@ -2893,8 +2894,7 @@ def __new__(cls, typename, bases, ns):
types = ns["__annotations__"]
field_names = list(types)
annotate = _make_eager_annotate(types)
- elif "__annotate__" in ns:
- original_annotate = ns["__annotate__"]
+ elif (original_annotate :=
_lazy_annotationlib.get_annotate_function(ns)) is not None:
types = _lazy_annotationlib.call_annotate_function(
original_annotate, _lazy_annotationlib.Format.FORWARDREF)
field_names = list(types)
@@ -3080,8 +3080,7 @@ def __new__(cls, name, bases, ns, total=True):
if "__annotations__" in ns:
own_annotate = None
own_annotations = ns["__annotations__"]
- elif "__annotate__" in ns:
- own_annotate = ns["__annotate__"]
+ elif (own_annotate := _lazy_annotationlib.get_annotate_function(ns))
is not None:
own_annotations = _lazy_annotationlib.call_annotate_function(
own_annotate, _lazy_annotationlib.Format.FORWARDREF,
owner=tp_dict
)
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-09-21-51-37.gh-issue-132261.gL8thm.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-09-21-51-37.gh-issue-132261.gL8thm.rst
new file mode 100644
index 00000000000000..0e58cf7f957dfd
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-09-21-51-37.gh-issue-132261.gL8thm.rst
@@ -0,0 +1,4 @@
+The internal storage for annotations and annotate functions on classes now
+uses different keys in the class dictionary. This eliminates various edge
+cases where access to the ``__annotate__`` and ``__annotations__``
+attributes would behave unpredictably.
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 75c23ddd91b1a1..b817ae6e68438b 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1915,10 +1915,17 @@ type_get_annotate(PyObject *tp, void
*Py_UNUSED(closure))
PyObject *annotate;
PyObject *dict = PyType_GetDict(type);
+ // First try __annotate__, in case that's been set explicitly
if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) < 0) {
Py_DECREF(dict);
return NULL;
}
+ if (!annotate) {
+ if (PyDict_GetItemRef(dict, &_Py_ID(__annotate_func__), &annotate) <
0) {
+ Py_DECREF(dict);
+ return NULL;
+ }
+ }
if (annotate) {
descrgetfunc get = Py_TYPE(annotate)->tp_descr_get;
if (get) {
@@ -1927,7 +1934,7 @@ type_get_annotate(PyObject *tp, void *Py_UNUSED(closure))
}
else {
annotate = Py_None;
- int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate);
+ int result = PyDict_SetItem(dict, &_Py_ID(__annotate_func__),
annotate);
if (result < 0) {
Py_DECREF(dict);
return NULL;
@@ -1959,13 +1966,13 @@ type_set_annotate(PyObject *tp, PyObject *value, void
*Py_UNUSED(closure))
PyObject *dict = PyType_GetDict(type);
assert(PyDict_Check(dict));
- int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), value);
+ int result = PyDict_SetItem(dict, &_Py_ID(__annotate_func__), value);
if (result < 0) {
Py_DECREF(dict);
return -1;
}
if (!Py_IsNone(value)) {
- if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) {
+ if (PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL) == -1) {
Py_DECREF(dict);
PyType_Modified(type);
return -1;
@@ -1987,10 +1994,18 @@ type_get_annotations(PyObject *tp, void
*Py_UNUSED(closure))
PyObject *annotations;
PyObject *dict = PyType_GetDict(type);
+ // First try __annotations__ (e.g. for "from __future__ import
annotations")
if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) {
Py_DECREF(dict);
return NULL;
}
+ if (!annotations) {
+ if (PyDict_GetItemRef(dict, &_Py_ID(__annotations_cache__),
&annotations) < 0) {
+ Py_DECREF(dict);
+ return NULL;
+ }
+ }
+
if (annotations) {
descrgetfunc get = Py_TYPE(annotations)->tp_descr_get;
if (get) {
@@ -1998,7 +2013,7 @@ type_get_annotations(PyObject *tp, void
*Py_UNUSED(closure))
}
}
else {
- PyObject *annotate = type_get_annotate(tp, NULL);
+ PyObject *annotate = PyObject_GetAttrString((PyObject *)type,
"__annotate__");
if (annotate == NULL) {
Py_DECREF(dict);
return NULL;
@@ -2026,7 +2041,7 @@ type_get_annotations(PyObject *tp, void
*Py_UNUSED(closure))
Py_DECREF(annotate);
if (annotations) {
int result = PyDict_SetItem(
- dict, &_Py_ID(__annotations__), annotations);
+ dict, &_Py_ID(__annotations_cache__), annotations);
if (result) {
Py_CLEAR(annotations);
} else {
@@ -2053,10 +2068,10 @@ type_set_annotations(PyObject *tp, PyObject *value,
void *Py_UNUSED(closure))
PyObject *dict = PyType_GetDict(type);
if (value != NULL) {
/* set */
- result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value);
+ result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value);
} else {
/* delete */
- result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL);
+ result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL);
if (result == 0) {
PyErr_SetString(PyExc_AttributeError, "__annotations__");
Py_DECREF(dict);
@@ -2067,6 +2082,11 @@ type_set_annotations(PyObject *tp, PyObject *value, void
*Py_UNUSED(closure))
Py_DECREF(dict);
return -1;
} else { // result can be 0 or 1
+ if (PyDict_Pop(dict, &_Py_ID(__annotate_func__), NULL) < 0) {
+ PyType_Modified(type);
+ Py_DECREF(dict);
+ return -1;
+ }
if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) {
PyType_Modified(type);
Py_DECREF(dict);
diff --git a/Python/codegen.c b/Python/codegen.c
index dc50737840f002..379d37c65ca8e6 100644
--- a/Python/codegen.c
+++ b/Python/codegen.c
@@ -815,7 +815,10 @@ codegen_process_deferred_annotations(compiler *c, location
loc)
Py_DECREF(conditional_annotation_indices);
RETURN_IF_ERROR(codegen_leave_annotations_scope(c, loc));
- RETURN_IF_ERROR(codegen_nameop(c, loc, &_Py_ID(__annotate__), Store));
+ RETURN_IF_ERROR(codegen_nameop(
+ c, loc,
+ ste->ste_type == ClassBlock ? &_Py_ID(__annotate_func__) :
&_Py_ID(__annotate__),
+ Store));
return SUCCESS;
error:
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
