https://github.com/python/cpython/commit/bc262de06b10a2d119c28bac75060bf00301697a
commit: bc262de06b10a2d119c28bac75060bf00301697a
branch: main
author: Mark Shannon <[email protected]>
committer: markshannon <[email protected]>
date: 2024-12-11T17:37:38Z
summary:
GH-125174: Mark objects as statically allocated. (#127797)
* Set a bit in the unused part of the refcount on 64 bit machines and the
free-threaded build.
* Use the top of the refcount range on 32 bit machines
files:
M Include/internal/pycore_object.h
M Include/object.h
M Include/refcount.h
M Lib/test/test_builtin.py
M Lib/test/test_capi/test_immortal.py
M Modules/_testinternalcapi.c
M Objects/object.c
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
index 6b0b464a6fdb96..22de3c9d4e32ea 100644
--- a/Include/internal/pycore_object.h
+++ b/Include/internal/pycore_object.h
@@ -73,14 +73,24 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *);
#define _PyObject_HEAD_INIT(type) \
{ \
.ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL, \
+ .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \
.ob_type = (type) \
}
#else
+#if SIZEOF_VOID_P > 4
#define _PyObject_HEAD_INIT(type) \
{ \
- .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \
+ .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \
+ .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \
.ob_type = (type) \
}
+#else
+#define _PyObject_HEAD_INIT(type) \
+ { \
+ .ob_refcnt = _Py_STATIC_IMMORTAL_INITIAL_REFCNT, \
+ .ob_type = (type) \
+ }
+#endif
#endif
#define _PyVarObject_HEAD_INIT(type, size) \
{ \
@@ -127,7 +137,11 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t
n)
_Py_AddRefTotal(_PyThreadState_GET(), n);
#endif
#if !defined(Py_GIL_DISABLED)
+#if SIZEOF_VOID_P > 4
+ op->ob_refcnt += (PY_UINT32_T)n;
+#else
op->ob_refcnt += n;
+#endif
#else
if (_Py_IsOwnedByCurrentThread(op)) {
uint32_t local = op->ob_ref_local;
diff --git a/Include/object.h b/Include/object.h
index 3876d8449afbe2..da7b3668c033f4 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -71,7 +71,7 @@ whose size is determined when the object is allocated.
#define PyObject_HEAD_INIT(type) \
{ \
0, \
- 0, \
+ _Py_STATICALLY_ALLOCATED_FLAG, \
{ 0 }, \
0, \
_Py_IMMORTAL_REFCNT_LOCAL, \
@@ -81,7 +81,7 @@ whose size is determined when the object is allocated.
#else
#define PyObject_HEAD_INIT(type) \
{ \
- { _Py_IMMORTAL_INITIAL_REFCNT }, \
+ { _Py_STATIC_IMMORTAL_INITIAL_REFCNT }, \
(type) \
},
#endif
@@ -120,9 +120,19 @@ struct _object {
__pragma(warning(disable: 4201))
#endif
union {
- Py_ssize_t ob_refcnt;
#if SIZEOF_VOID_P > 4
- PY_UINT32_T ob_refcnt_split[2];
+ PY_INT64_T ob_refcnt_full; /* This field is needed for efficient
initialization with Clang on ARM */
+ struct {
+# if PY_BIG_ENDIAN
+ PY_UINT32_T ob_flags;
+ PY_UINT32_T ob_refcnt;
+# else
+ PY_UINT32_T ob_refcnt;
+ PY_UINT32_T ob_flags;
+# endif
+ };
+#else
+ Py_ssize_t ob_refcnt;
#endif
};
#ifdef _MSC_VER
@@ -142,7 +152,7 @@ struct _object {
// trashcan mechanism as a linked list pointer and by the GC to store the
// computed "gc_refs" refcount.
uintptr_t ob_tid;
- uint16_t _padding;
+ uint16_t ob_flags;
PyMutex ob_mutex; // per-object lock
uint8_t ob_gc_bits; // gc-related state
uint32_t ob_ref_local; // local reference count
diff --git a/Include/refcount.h b/Include/refcount.h
index 141cbd34dd72e6..6908c426141378 100644
--- a/Include/refcount.h
+++ b/Include/refcount.h
@@ -19,6 +19,9 @@ immortal. The latter should be the only instances that require
cleanup during runtime finalization.
*/
+/* Leave the low bits for refcount overflow for old stable ABI code */
+#define _Py_STATICALLY_ALLOCATED_FLAG (1 << 7)
+
#if SIZEOF_VOID_P > 4
/*
In 64+ bit systems, any object whose 32 bit reference count is >= 2**31
@@ -39,7 +42,8 @@ beyond the refcount limit. Immortality checks for reference
count decreases will
be done by checking the bit sign flag in the lower 32 bits.
*/
-#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(3UL << 30))
+#define _Py_IMMORTAL_INITIAL_REFCNT (3UL << 30)
+#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT
((Py_ssize_t)(_Py_IMMORTAL_INITIAL_REFCNT |
(((Py_ssize_t)_Py_STATICALLY_ALLOCATED_FLAG) << 32)))
#else
/*
@@ -54,8 +58,10 @@ immortality, but the execution would still be correct.
Reference count increases and decreases will first go through an immortality
check by comparing the reference count field to the minimum immortality
refcount.
*/
-#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(3L << 29))
+#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(5L << 28))
#define _Py_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(1L << 30))
+#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(7L << 28))
+#define _Py_STATIC_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(6L << 28))
#endif
// Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`,
which is
@@ -123,10 +129,21 @@ static inline Py_ALWAYS_INLINE int
_Py_IsImmortal(PyObject *op)
#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op))
+static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op)
+{
+#if defined(Py_GIL_DISABLED) || SIZEOF_VOID_P > 4
+ return (op->ob_flags & _Py_STATICALLY_ALLOCATED_FLAG) != 0;
+#else
+ return op->ob_refcnt >= _Py_STATIC_IMMORTAL_MINIMUM_REFCNT;
+#endif
+}
+#define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op))
+
// Py_SET_REFCNT() implementation for stable ABI
PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt);
static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) {
+ assert(refcnt >= 0);
#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000
// Stable ABI implements Py_SET_REFCNT() as a function call
// on limited C API version 3.13 and newer.
@@ -139,9 +156,12 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t
refcnt) {
if (_Py_IsImmortal(ob)) {
return;
}
-
#ifndef Py_GIL_DISABLED
+#if SIZEOF_VOID_P > 4
+ ob->ob_refcnt = (PY_UINT32_T)refcnt;
+#else
ob->ob_refcnt = refcnt;
+#endif
#else
if (_Py_IsOwnedByCurrentThread(ob)) {
if ((size_t)refcnt > (size_t)UINT32_MAX) {
@@ -252,13 +272,13 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject
*op)
_Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT));
}
#elif SIZEOF_VOID_P > 4
- PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN];
+ PY_UINT32_T cur_refcnt = op->ob_refcnt;
if (((int32_t)cur_refcnt) < 0) {
// the object is immortal
_Py_INCREF_IMMORTAL_STAT_INC();
return;
}
- op->ob_refcnt_split[PY_BIG_ENDIAN] = cur_refcnt + 1;
+ op->ob_refcnt = cur_refcnt + 1;
#else
if (_Py_IsImmortal(op)) {
_Py_INCREF_IMMORTAL_STAT_INC();
@@ -354,7 +374,13 @@ static inline void Py_DECREF(PyObject *op)
#elif defined(Py_REF_DEBUG)
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
{
+#if SIZEOF_VOID_P > 4
+ /* If an object has been freed, it will have a negative full refcnt
+ * If it has not it been freed, will have a very large refcnt */
+ if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (UINT32_MAX - (1<<20))) {
+#else
if (op->ob_refcnt <= 0) {
+#endif
_Py_NegativeRefcount(filename, lineno, op);
}
if (_Py_IsImmortal(op)) {
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index e51711d9b4f1a4..06df217881a52f 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -2691,7 +2691,7 @@ def __del__(self):
class ImmortalTests(unittest.TestCase):
if sys.maxsize < (1 << 32):
- IMMORTAL_REFCOUNT = 3 << 29
+ IMMORTAL_REFCOUNT = 7 << 28
else:
IMMORTAL_REFCOUNT = 3 << 30
diff --git a/Lib/test/test_capi/test_immortal.py
b/Lib/test/test_capi/test_immortal.py
index ef5d32b7f01935..3e36913ac301c3 100644
--- a/Lib/test/test_capi/test_immortal.py
+++ b/Lib/test/test_capi/test_immortal.py
@@ -2,6 +2,7 @@
from test.support import import_helper
_testcapi = import_helper.import_module('_testcapi')
+_testinternalcapi = import_helper.import_module('_testinternalcapi')
class TestCAPI(unittest.TestCase):
@@ -11,6 +12,21 @@ def test_immortal_builtins(self):
def test_immortal_small_ints(self):
_testcapi.test_immortal_small_ints()
+class TestInternalCAPI(unittest.TestCase):
+
+ def test_immortal_builtins(self):
+ for obj in range(-5, 256):
+ self.assertTrue(_testinternalcapi.is_static_immortal(obj))
+ self.assertTrue(_testinternalcapi.is_static_immortal(None))
+ self.assertTrue(_testinternalcapi.is_static_immortal(False))
+ self.assertTrue(_testinternalcapi.is_static_immortal(True))
+ self.assertTrue(_testinternalcapi.is_static_immortal(...))
+ self.assertTrue(_testinternalcapi.is_static_immortal(()))
+ for obj in range(300, 400):
+ self.assertFalse(_testinternalcapi.is_static_immortal(obj))
+ for obj in ([], {}, set()):
+ self.assertFalse(_testinternalcapi.is_static_immortal(obj))
+
if __name__ == "__main__":
unittest.main()
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 288daf09a5fe5c..014f89997f7f60 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -2049,6 +2049,15 @@ get_tracked_heap_size(PyObject *self, PyObject
*Py_UNUSED(ignored))
return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size);
}
+static PyObject *
+is_static_immortal(PyObject *self, PyObject *op)
+{
+ if (_Py_IsStaticImmortal(op)) {
+ Py_RETURN_TRUE;
+ }
+ Py_RETURN_FALSE;
+}
+
static PyMethodDef module_functions[] = {
{"get_configs", get_configs, METH_NOARGS},
{"get_recursion_depth", get_recursion_depth, METH_NOARGS},
@@ -2146,6 +2155,7 @@ static PyMethodDef module_functions[] = {
{"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS},
{"has_deferred_refcount", has_deferred_refcount, METH_O},
{"get_tracked_heap_size", get_tracked_heap_size, METH_NOARGS},
+ {"is_static_immortal", is_static_immortal, METH_O},
{NULL, NULL} /* sentinel */
};
diff --git a/Objects/object.c b/Objects/object.c
index 74f47fa4239032..c64675b5e1d6c2 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -2475,10 +2475,16 @@ new_reference(PyObject *op)
{
// Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1
#if !defined(Py_GIL_DISABLED)
+#if SIZEOF_VOID_P > 4
+ op->ob_refcnt_full = 1;
+ assert(op->ob_refcnt == 1);
+ assert(op->ob_flags == 0);
+#else
op->ob_refcnt = 1;
+#endif
#else
op->ob_tid = _Py_ThreadId();
- op->_padding = 0;
+ op->ob_flags = 0;
op->ob_mutex = (PyMutex){ 0 };
op->ob_gc_bits = 0;
op->ob_ref_local = 1;
@@ -2515,6 +2521,10 @@ _Py_SetImmortalUntracked(PyObject *op)
|| PyUnicode_CHECK_INTERNED(op) ==
SSTATE_INTERNED_IMMORTAL_STATIC);
}
#endif
+ // Check if already immortal to avoid degrading from static immortal to
plain immortal
+ if (_Py_IsImmortal(op)) {
+ return;
+ }
#ifdef Py_GIL_DISABLED
op->ob_tid = _Py_UNOWNED_TID;
op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL;
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
