https://github.com/python/cpython/commit/c32dc47aca6e8fac152699bc613e015c44ccdba9
commit: c32dc47aca6e8fac152699bc613e015c44ccdba9
branch: main
author: Mark Shannon <[email protected]>
committer: markshannon <[email protected]>
date: 2024-04-02T11:59:21+01:00
summary:
GH-115776: Embed the values array into the object, for "normal" Python objects.
(GH-116115)
files:
A Misc/NEWS.d/next/Core and
Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst
A Objects/object_layout_313.gv
A Objects/object_layout_313.png
A Objects/object_layout_full_313.gv
A Objects/object_layout_full_313.png
M Include/cpython/pystats.h
M Include/internal/pycore_code.h
M Include/internal/pycore_dict.h
M Include/internal/pycore_object.h
M Include/internal/pycore_opcode_metadata.h
M Include/internal/pycore_uop_ids.h
M Include/internal/pycore_uop_metadata.h
M Include/object.h
M Lib/test/test_capi/test_mem.py
M Lib/test/test_class.py
M Lib/test/test_opcache.py
M Modules/_testbuffer.c
M Modules/_testcapimodule.c
M Modules/_testinternalcapi.c
M Objects/dictobject.c
M Objects/object.c
M Objects/object_layout.md
M Objects/object_layout_312.gv
M Objects/object_layout_312.png
M Objects/typeobject.c
M Python/bytecodes.c
M Python/executor_cases.c.h
M Python/gc.c
M Python/gc_free_threading.c
M Python/generated_cases.c.h
M Python/optimizer_cases.c.h
M Python/specialize.c
M Tools/cases_generator/analyzer.py
M Tools/gdb/libpython.py
M Tools/scripts/summarize_stats.py
diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h
index 2fb7723f583cc7..e74fdd4d32e26c 100644
--- a/Include/cpython/pystats.h
+++ b/Include/cpython/pystats.h
@@ -77,12 +77,11 @@ typedef struct _object_stats {
uint64_t frees;
uint64_t to_freelist;
uint64_t from_freelist;
- uint64_t new_values;
+ uint64_t inline_values;
uint64_t dict_materialized_on_request;
uint64_t dict_materialized_new_key;
uint64_t dict_materialized_too_big;
uint64_t dict_materialized_str_subclass;
- uint64_t dict_dematerialized;
uint64_t type_cache_hits;
uint64_t type_cache_misses;
uint64_t type_cache_dunder_hits;
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index e004783ee48198..6c90c9e284103c 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -79,7 +79,10 @@ typedef struct {
typedef struct {
uint16_t counter;
uint16_t type_version[2];
- uint16_t keys_version[2];
+ union {
+ uint16_t keys_version[2];
+ uint16_t dict_offset;
+ };
uint16_t descr[4];
} _PyLoadMethodCache;
diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h
index ef59960dbab071..5507bdd539cc42 100644
--- a/Include/internal/pycore_dict.h
+++ b/Include/internal/pycore_dict.h
@@ -11,7 +11,7 @@ extern "C" {
#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_identifier.h" // _Py_Identifier
-#include "pycore_object.h" // PyDictOrValues
+#include "pycore_object.h" // PyManagedDictPointer
// Unsafe flavor of PyDict_GetItemWithError(): no error checking
extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
@@ -181,6 +181,10 @@ struct _dictkeysobject {
* [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order.
*/
struct _dictvalues {
+ uint8_t capacity;
+ uint8_t size;
+ uint8_t embedded;
+ uint8_t valid;
PyObject *values[1];
};
@@ -196,6 +200,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
size_t index = (size_t)1 << dk->dk_log2_index_bytes;
return (&indices[index]);
}
+
static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
assert(dk->dk_kind == DICT_KEYS_GENERAL);
return (PyDictKeyEntry*)_DK_ENTRIES(dk);
@@ -211,9 +216,6 @@ static inline PyDictUnicodeEntry*
DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
#define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1)
#define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS +
DICT_WATCHED_MUTATION_BITS)) - 1)
-#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1]
-#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2]
-
#ifdef Py_GIL_DISABLED
#define DICT_NEXT_VERSION(INTERP) \
(_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version,
DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
@@ -246,25 +248,63 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag &
DICT_WATCHER_AND_MODIFICATION_MASK);
}
-extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj,
PyDictValues *values);
-PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj,
PyDictOrValues *dorv);
+extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj);
+
PyAPI_FUNC(PyObject *)_PyDict_FromItems(
PyObject *const *keys, Py_ssize_t keys_offset,
PyObject *const *values, Py_ssize_t values_offset,
Py_ssize_t length);
+static inline uint8_t *
+get_insertion_order_array(PyDictValues *values)
+{
+ return (uint8_t *)&values->values[values->capacity];
+}
+
static inline void
_PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix)
{
assert(ix < SHARED_KEYS_MAX_SIZE);
- uint8_t *size_ptr = ((uint8_t *)values)-2;
- int size = *size_ptr;
- assert(size+2 < DICT_VALUES_SIZE(values));
- size++;
- size_ptr[-size] = (uint8_t)ix;
- *size_ptr = size;
+ int size = values->size;
+ uint8_t *array = get_insertion_order_array(values);
+ assert(size < values->capacity);
+ assert(((uint8_t)ix) == ix);
+ array[size] = (uint8_t)ix;
+ values->size = size+1;
+}
+
+static inline size_t
+shared_keys_usable_size(PyDictKeysObject *keys)
+{
+#ifdef Py_GIL_DISABLED
+ // dk_usable will decrease for each instance that is created and each
+ // value that is added. dk_nentries will increase for each value that
+ // is added. We want to always return the right value or larger.
+ // We therefore increase dk_nentries first and we decrease dk_usable
+ // second, and conversely here we read dk_usable first and dk_entries
+ // second (to avoid the case where we read entries before the increment
+ // and read usable after the decrement)
+ return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
+ _Py_atomic_load_ssize_acquire(&keys->dk_nentries));
+#else
+ return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
+#endif
}
+static inline size_t
+_PyInlineValuesSize(PyTypeObject *tp)
+{
+ PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys;
+ assert(keys != NULL);
+ size_t size = shared_keys_usable_size(keys);
+ size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *));
+ assert(prefix_size < 256);
+ return prefix_size + (size + 1) * sizeof(PyObject *);
+}
+
+int
+_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj);
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
index 0b17ddf0c973ef..4fc5e9bf653c1c 100644
--- a/Include/internal/pycore_object.h
+++ b/Include/internal/pycore_object.h
@@ -265,9 +265,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
{
assert(op != NULL);
Py_SET_TYPE(op, typeobj);
- if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) {
- Py_INCREF(typeobj);
- }
+ assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) ||
_Py_IsImmortal(typeobj));
+ Py_INCREF(typeobj);
_Py_NewReference(op);
}
@@ -611,8 +610,7 @@ extern PyTypeObject*
_PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *);
extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const
char *, int);
-extern int _PyObject_InitializeDict(PyObject *obj);
-int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
+void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues
*values,
PyObject *name, PyObject *value);
PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
@@ -627,46 +625,26 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj,
PyDictValues *values,
#endif
typedef union {
- PyObject *dict;
- /* Use a char* to generate a warning if directly assigning a PyDictValues
*/
- char *values;
-} PyDictOrValues;
+ PyDictObject *dict;
+} PyManagedDictPointer;
-static inline PyDictOrValues *
-_PyObject_DictOrValuesPointer(PyObject *obj)
+static inline PyManagedDictPointer *
+_PyObject_ManagedDictPointer(PyObject *obj)
{
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET);
-}
-
-static inline int
-_PyDictOrValues_IsValues(PyDictOrValues dorv)
-{
- return ((uintptr_t)dorv.values) & 1;
+ return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET);
}
static inline PyDictValues *
-_PyDictOrValues_GetValues(PyDictOrValues dorv)
-{
- assert(_PyDictOrValues_IsValues(dorv));
- return (PyDictValues *)(dorv.values + 1);
-}
-
-static inline PyObject *
-_PyDictOrValues_GetDict(PyDictOrValues dorv)
+_PyObject_InlineValues(PyObject *obj)
{
- assert(!_PyDictOrValues_IsValues(dorv));
- return dorv.dict;
-}
-
-static inline void
-_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values)
-{
- ptr->values = ((char *)values) - 1;
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject));
+ return (PyDictValues *)((char *)obj + sizeof(PyObject));
}
extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);
-extern void _PyObject_FreeInstanceAttributes(PyObject *obj);
extern int _PyObject_IsInstanceDictEmpty(PyObject *);
// Export for 'math' shared extension
diff --git a/Include/internal/pycore_opcode_metadata.h
b/Include/internal/pycore_opcode_metadata.h
index de525f72d3523e..aa87dc413876f0 100644
--- a/Include/internal/pycore_opcode_metadata.h
+++ b/Include/internal/pycore_opcode_metadata.h
@@ -1085,7 +1085,7 @@ const struct opcode_metadata
_PyOpcode_opcode_metadata[268] = {
[LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000,
HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG |
HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG |
HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
- [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG |
HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
+ [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG |
HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG |
HAS_ESCAPES_FLAG },
[LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG |
HAS_PURE_FLAG },
[LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG |
HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@@ -1269,7 +1269,7 @@ _PyOpcode_macro_expansion[256] = {
[LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } },
[LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, {
_LOAD_ATTR_CLASS, 4, 5 } } },
[LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { {
_GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, {
_LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } },
- [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { {
_GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, {
_LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } },
+ [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { {
_GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, {
_LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } },
[LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { {
_GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } },
[LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { {
_GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 },
{ _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } },
[LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 },
{ _LOAD_ATTR_MODULE, 1, 3 } } },
@@ -1316,7 +1316,7 @@ _PyOpcode_macro_expansion[256] = {
[SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { {
_SET_FUNCTION_ATTRIBUTE, 0, 0 } } },
[SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } },
[STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } },
- [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { {
_GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, {
_STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
+ [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { {
_GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, {
_STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
[STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 },
{ _STORE_ATTR_SLOT, 1, 3 } } },
[STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } },
[STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } },
diff --git a/Include/internal/pycore_uop_ids.h
b/Include/internal/pycore_uop_ids.h
index bcb10ab723ecba..54dc6dcf408116 100644
--- a/Include/internal/pycore_uop_ids.h
+++ b/Include/internal/pycore_uop_ids.h
@@ -109,7 +109,7 @@ extern "C" {
#define _GUARD_BOTH_INT 347
#define _GUARD_BOTH_UNICODE 348
#define _GUARD_BUILTINS_VERSION 349
-#define _GUARD_DORV_VALUES 350
+#define _GUARD_DORV_NO_DICT 350
#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351
#define _GUARD_GLOBALS_VERSION 352
#define _GUARD_IS_FALSE_POP 353
diff --git a/Include/internal/pycore_uop_metadata.h
b/Include/internal/pycore_uop_metadata.h
index 51206cd4ca2fdf..0f2046fb3d0c3d 100644
--- a/Include/internal/pycore_uop_metadata.h
+++ b/Include/internal/pycore_uop_metadata.h
@@ -136,8 +136,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG |
HAS_OPARG_AND_1_FLAG,
[_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
- [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG |
HAS_PASSTHROUGH_FLAG,
- [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG |
HAS_ESCAPES_FLAG,
+ [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG,
[_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG,
@@ -145,7 +145,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_LOAD_ATTR_CLASS_0] = 0,
[_LOAD_ATTR_CLASS_1] = 0,
[_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG,
- [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
+ [_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_STORE_ATTR_INSTANCE_VALUE] = 0,
[_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG,
[_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
@@ -342,7 +342,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_GUARD_BOTH_INT] = "_GUARD_BOTH_INT",
[_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE",
[_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION",
- [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES",
+ [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT",
[_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] =
"_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT",
[_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION",
[_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP",
@@ -736,7 +736,7 @@ int _PyUop_num_popped(int opcode, int oparg)
return 1;
case _LOAD_ATTR_CLASS:
return 1;
- case _GUARD_DORV_VALUES:
+ case _GUARD_DORV_NO_DICT:
return 1;
case _STORE_ATTR_INSTANCE_VALUE:
return 2;
diff --git a/Include/object.h b/Include/object.h
index 96790844a7b9f0..13443329dfb5a2 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -629,13 +629,18 @@ given type object has a specified feature.
/* Track types initialized using _PyStaticType_InitBuiltin(). */
#define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1)
+/* The values array is placed inline directly after the rest of
+ * the object. Implies Py_TPFLAGS_HAVE_GC.
+ */
+#define Py_TPFLAGS_INLINE_VALUES (1 << 2)
+
/* Placement of weakref pointers are managed by the VM, not by the type.
* The VM will automatically set tp_weaklistoffset.
*/
#define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3)
/* Placement of dict (and values) pointers are managed by the VM, not by the
type.
- * The VM will automatically set tp_dictoffset.
+ * The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC.
*/
#define Py_TPFLAGS_MANAGED_DICT (1 << 4)
diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py
index 04f17a9ec9e72a..1958ecc0df4a7c 100644
--- a/Lib/test/test_capi/test_mem.py
+++ b/Lib/test/test_capi/test_mem.py
@@ -148,8 +148,8 @@ class C(): pass
self.assertIn(b'MemoryError', out)
*_, count = line.split(b' ')
count = int(count)
- self.assertLessEqual(count, i*5)
- self.assertGreaterEqual(count, i*5-2)
+ self.assertLessEqual(count, i*10)
+ self.assertGreaterEqual(count, i*10-4)
# Py_GIL_DISABLED requires mimalloc (not malloc)
diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py
index 0cf06243dc8da0..4c1814142736e3 100644
--- a/Lib/test/test_class.py
+++ b/Lib/test/test_class.py
@@ -2,7 +2,6 @@
import unittest
-
testmeths = [
# Binary operations
@@ -789,5 +788,81 @@ def __init__(self, obj):
self.assertEqual(calls, 100)
+from _testinternalcapi import has_inline_values
+
+Py_TPFLAGS_MANAGED_DICT = (1 << 2)
+
+class Plain:
+ pass
+
+
+class WithAttrs:
+
+ def __init__(self):
+ self.a = 1
+ self.b = 2
+ self.c = 3
+ self.d = 4
+
+
+class TestInlineValues(unittest.TestCase):
+
+ def test_flags(self):
+ self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT,
Py_TPFLAGS_MANAGED_DICT)
+ self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT,
Py_TPFLAGS_MANAGED_DICT)
+
+ def test_has_inline_values(self):
+ c = Plain()
+ self.assertTrue(has_inline_values(c))
+ del c.__dict__
+ self.assertFalse(has_inline_values(c))
+
+ def test_instances(self):
+ self.assertTrue(has_inline_values(Plain()))
+ self.assertTrue(has_inline_values(WithAttrs()))
+
+ def test_inspect_dict(self):
+ for cls in (Plain, WithAttrs):
+ c = cls()
+ c.__dict__
+ self.assertTrue(has_inline_values(c))
+
+ def test_update_dict(self):
+ d = { "e": 5, "f": 6 }
+ for cls in (Plain, WithAttrs):
+ c = cls()
+ c.__dict__.update(d)
+ self.assertTrue(has_inline_values(c))
+
+ @staticmethod
+ def set_100(obj):
+ for i in range(100):
+ setattr(obj, f"a{i}", i)
+
+ def check_100(self, obj):
+ for i in range(100):
+ self.assertEqual(getattr(obj, f"a{i}"), i)
+
+ def test_many_attributes(self):
+ class C: pass
+ c = C()
+ self.assertTrue(has_inline_values(c))
+ self.set_100(c)
+ self.assertFalse(has_inline_values(c))
+ self.check_100(c)
+ c = C()
+ self.assertTrue(has_inline_values(c))
+
+ def test_many_attributes_with_dict(self):
+ class C: pass
+ c = C()
+ d = c.__dict__
+ self.assertTrue(has_inline_values(c))
+ self.set_100(c)
+ self.assertFalse(has_inline_values(c))
+ self.check_100(c)
+
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py
index 5fb4b815c95d07..8829c9a6d88261 100644
--- a/Lib/test/test_opcache.py
+++ b/Lib/test/test_opcache.py
@@ -1044,20 +1044,13 @@ def test_dict_materialization(self):
c.a = 1
c.b = 2
c.__dict__
- self.assertIs(
- _testinternalcapi.get_object_dict_values(c),
- None
- )
+ self.assertEqual(c.__dict__, {"a":1, "b": 2})
def test_dict_dematerialization(self):
c = C()
c.a = 1
c.b = 2
c.__dict__
- self.assertIs(
- _testinternalcapi.get_object_dict_values(c),
- None
- )
for _ in range(100):
c.a
self.assertEqual(
@@ -1072,10 +1065,6 @@ def test_dict_dematerialization_multiple_refs(self):
d = c.__dict__
for _ in range(100):
c.a
- self.assertIs(
- _testinternalcapi.get_object_dict_values(c),
- None
- )
self.assertIs(c.__dict__, d)
def test_dict_dematerialization_copy(self):
diff --git a/Misc/NEWS.d/next/Core and
Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst b/Misc/NEWS.d/next/Core
and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst
new file mode 100644
index 00000000000000..5974b1882acb22
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and
Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst
@@ -0,0 +1,4 @@
+The array of values, the ``PyDictValues`` struct is now embedded in the
+object during allocation. This provides better performance in the common
+case, and does not degrade as much when the object's ``__dict__`` is
+materialized.
diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c
index 5084bcadb10f85..cad21bdb4d85ed 100644
--- a/Modules/_testbuffer.c
+++ b/Modules/_testbuffer.c
@@ -2820,6 +2820,9 @@ static int
_testbuffer_exec(PyObject *mod)
{
Py_SET_TYPE(&NDArray_Type, &PyType_Type);
+ if (PyType_Ready(&NDArray_Type)) {
+ return -1;
+ }
if (PyModule_AddType(mod, &NDArray_Type) < 0) {
return -1;
}
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index e9db6e5683e344..b2af47d05ee196 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -3869,7 +3869,9 @@ PyInit__testcapi(void)
return NULL;
Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type);
-
+ if (PyType_Ready(&_HashInheritanceTester_Type) < 0) {
+ return NULL;
+ }
if (PyType_Ready(&matmulType) < 0)
return NULL;
Py_INCREF(&matmulType);
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index c07652facc0ae2..d6d50e75b612df 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -15,7 +15,7 @@
#include "pycore_ceval.h" // _PyEval_AddPendingCall()
#include "pycore_compile.h" // _PyCompile_CodeGen()
#include "pycore_context.h" // _PyContext_NewHamtForTests()
-#include "pycore_dict.h" // _PyDictOrValues_GetValues()
+#include "pycore_dict.h" // _PyManagedDictPointer_GetValues()
#include "pycore_fileutils.h" // _Py_normpath()
#include "pycore_frame.h" // _PyInterpreterFrame
#include "pycore_gc.h" // PyGC_Head
@@ -1297,14 +1297,13 @@ static PyObject *
get_object_dict_values(PyObject *self, PyObject *obj)
{
PyTypeObject *type = Py_TYPE(obj);
- if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) {
+ if (!_PyType_HasFeature(type, Py_TPFLAGS_INLINE_VALUES)) {
Py_RETURN_NONE;
}
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
- if (!_PyDictOrValues_IsValues(dorv)) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (!values->valid) {
Py_RETURN_NONE;
}
- PyDictValues *values = _PyDictOrValues_GetValues(dorv);
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(keys != NULL);
int size = (int)keys->dk_nentries;
@@ -1784,6 +1783,16 @@ get_py_thread_id(PyObject *self, PyObject
*Py_UNUSED(ignored))
}
#endif
+static PyObject *
+has_inline_values(PyObject *self, PyObject *obj)
+{
+ if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
+ _PyObject_InlineValues(obj)->valid) {
+ Py_RETURN_TRUE;
+ }
+ Py_RETURN_FALSE;
+}
+
static PyMethodDef module_functions[] = {
{"get_configs", get_configs, METH_NOARGS},
{"get_recursion_depth", get_recursion_depth, METH_NOARGS},
@@ -1857,6 +1866,7 @@ static PyMethodDef module_functions[] = {
_TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF
{"get_rare_event_counters", get_rare_event_counters, METH_NOARGS},
{"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS},
+ {"has_inline_values", has_inline_values, METH_O},
#ifdef Py_GIL_DISABLED
{"py_thread_id", get_py_thread_id, METH_NOARGS},
#endif
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 536746ca41eed5..58a3d979339c49 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -361,6 +361,10 @@ static int
dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject
*default_value,
PyObject **result, int incref_result);
+#ifndef NDEBUG
+static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj);
+#endif
+
#include "clinic/dictobject.c.h"
@@ -624,8 +628,9 @@ static inline int
get_index_from_order(PyDictObject *mp, Py_ssize_t i)
{
assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
- assert(i < (((char *)mp->ma_values)[-2]));
- return ((char *)mp->ma_values)[-3-i];
+ assert(i < mp->ma_values->size);
+ uint8_t *array = get_insertion_order_array(mp->ma_values);
+ return array[i];
}
#ifdef DEBUG_PYDICT
@@ -672,6 +677,10 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
else {
CHECK(keys->dk_kind == DICT_KEYS_SPLIT);
CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
+ if (mp->ma_values->embedded) {
+ CHECK(mp->ma_values->embedded == 1);
+ CHECK(mp->ma_values->valid == 1);
+ }
}
if (check_content) {
@@ -821,33 +830,44 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
PyMem_Free(keys);
}
+static size_t
+values_size_from_count(size_t count)
+{
+ assert(count >= 1);
+ size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *));
+ assert(suffix_size < 128);
+ assert(suffix_size % sizeof(PyObject *) == 0);
+ return (count + 1) * sizeof(PyObject *) + suffix_size;
+}
+
+#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys)
+
static inline PyDictValues*
new_values(size_t size)
{
- assert(size >= 1);
- size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *));
- assert(prefix_size < 256);
- size_t n = prefix_size + size * sizeof(PyObject *);
- uint8_t *mem = PyMem_Malloc(n);
- if (mem == NULL) {
+ size_t n = values_size_from_count(size);
+ PyDictValues *res = (PyDictValues *)PyMem_Malloc(n);
+ if (res == NULL) {
return NULL;
}
- assert(prefix_size % sizeof(PyObject *) == 0);
- mem[prefix_size-1] = (uint8_t)prefix_size;
- return (PyDictValues*)(mem + prefix_size);
+ res->embedded = 0;
+ res->size = 0;
+ assert(size < 256);
+ res->capacity = (uint8_t)size;
+ return res;
}
static inline void
free_values(PyDictValues *values, bool use_qsbr)
{
- int prefix_size = DICT_VALUES_SIZE(values);
+ assert(values->embedded == 0);
#ifdef Py_GIL_DISABLED
if (use_qsbr) {
- _PyMem_FreeDelayed(((char *)values)-prefix_size);
+ _PyMem_FreeDelayed(values);
return;
}
#endif
- PyMem_Free(((char *)values)-prefix_size);
+ PyMem_Free(values);
}
/* Consumes a reference to the keys object */
@@ -887,24 +907,6 @@ new_dict(PyInterpreterState *interp,
return (PyObject *)mp;
}
-static inline size_t
-shared_keys_usable_size(PyDictKeysObject *keys)
-{
-#ifdef Py_GIL_DISABLED
- // dk_usable will decrease for each instance that is created and each
- // value that is added. dk_nentries will increase for each value that
- // is added. We want to always return the right value or larger.
- // We therefore increase dk_nentries first and we decrease dk_usable
- // second, and conversely here we read dk_usable first and dk_entries
- // second (to avoid the case where we read entries before the increment
- // and read usable after the decrement)
- return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
- _Py_atomic_load_ssize_acquire(&keys->dk_nentries));
-#else
- return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
-#endif
-}
-
/* Consumes a reference to the keys object */
static PyObject *
new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys)
@@ -915,7 +917,6 @@ new_dict_with_shared_keys(PyInterpreterState *interp,
PyDictKeysObject *keys)
dictkeys_decref(interp, keys, false);
return PyErr_NoMemory();
}
- ((char *)values)[-2] = 0;
for (size_t i = 0; i < size; i++) {
values->values[i] = NULL;
}
@@ -1419,7 +1420,7 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject
*key, Py_hash_t hash, PyOb
if (values == NULL)
goto read_failed;
- uint8_t capacity =
_Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values));
+ uint8_t capacity =
_Py_atomic_load_uint8_relaxed(&values->capacity);
if (ix >= (Py_ssize_t)capacity)
goto read_failed;
@@ -1525,6 +1526,7 @@ _PyDict_MaybeUntrack(PyObject *op)
return;
mp = (PyDictObject *) op;
+ ASSERT_CONSISTENT(mp);
numentries = mp->ma_keys->dk_nentries;
if (_PyDict_HasSplitTable(mp)) {
for (i = 0; i < numentries; i++) {
@@ -1945,7 +1947,14 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp,
set_keys(mp, newkeys);
dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp));
set_values(mp, NULL);
- free_values(oldvalues, IS_DICT_SHARED(mp));
+ if (oldvalues->embedded) {
+ assert(oldvalues->embedded == 1);
+ assert(oldvalues->valid == 1);
+ oldvalues->valid = 0;
+ }
+ else {
+ free_values(oldvalues, IS_DICT_SHARED(mp));
+ }
}
else { // oldkeys is combined.
if (oldkeys->dk_kind == DICT_KEYS_GENERAL) {
@@ -2464,17 +2473,19 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key,
PyObject *value,
static void
delete_index_from_values(PyDictValues *values, Py_ssize_t ix)
{
- uint8_t *size_ptr = ((uint8_t *)values)-2;
- int size = *size_ptr;
+ uint8_t *array = get_insertion_order_array(values);
+ int size = values->size;
+ assert(size <= values->capacity);
int i;
- for (i = 1; size_ptr[-i] != ix; i++) {
- assert(i <= size);
+ for (i = 0; array[i] != ix; i++) {
+ assert(i < size);
}
- assert(i <= size);
+ assert(i < size);
+ size--;
for (; i < size; i++) {
- size_ptr[-i] = size_ptr[-i-1];
+ array[i] = array[i+1];
}
- *size_ptr = size -1;
+ values->size = size;
}
static int
@@ -2669,10 +2680,12 @@ clear_lock_held(PyObject *op)
mp->ma_version_tag = new_version;
/* ...then clear the keys and values */
if (oldvalues != NULL) {
- n = oldkeys->dk_nentries;
- for (i = 0; i < n; i++)
- Py_CLEAR(oldvalues->values[i]);
- free_values(oldvalues, IS_DICT_SHARED(mp));
+ if (!oldvalues->embedded) {
+ n = oldkeys->dk_nentries;
+ for (i = 0; i < n; i++)
+ Py_CLEAR(oldvalues->values[i]);
+ free_values(oldvalues, IS_DICT_SHARED(mp));
+ }
dictkeys_decref(interp, oldkeys, false);
}
else {
@@ -3059,10 +3072,12 @@ dict_dealloc(PyObject *self)
PyObject_GC_UnTrack(mp);
Py_TRASHCAN_BEGIN(mp, dict_dealloc)
if (values != NULL) {
- for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) {
- Py_XDECREF(values->values[i]);
+ if (values->embedded == 0) {
+ for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) {
+ Py_XDECREF(values->values[i]);
+ }
+ free_values(values, false);
}
- free_values(values, false);
dictkeys_decref(interp, keys, false);
}
else if (keys != NULL) {
@@ -3595,10 +3610,12 @@ dict_dict_merge(PyInterpreterState *interp,
PyDictObject *mp, PyDictObject *othe
PyDictKeysObject *okeys = other->ma_keys;
// If other is clean, combined, and just allocated, just clone it.
- if (other->ma_values == NULL &&
- other->ma_used == okeys->dk_nentries &&
- (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
- USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) {
+ if (mp->ma_values == NULL &&
+ other->ma_values == NULL &&
+ other->ma_used == okeys->dk_nentries &&
+ (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
+ USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)
+ ) {
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL);
PyDictKeysObject *keys = clone_combined_dict_keys(other);
@@ -3608,11 +3625,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject
*mp, PyDictObject *othe
ensure_shared_on_resize(mp);
dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp));
mp->ma_keys = keys;
- if (_PyDict_HasSplitTable(mp)) {
- free_values(mp->ma_values, IS_DICT_SHARED(mp));
- mp->ma_values = NULL;
- }
-
mp->ma_used = other->ma_used;
mp->ma_version_tag = new_version;
ASSERT_CONSISTENT(mp);
@@ -3816,6 +3828,27 @@ dict_copy_impl(PyDictObject *self)
return PyDict_Copy((PyObject *)self);
}
+/* Copies the values, but does not change the reference
+ * counts of the objects in the array. */
+static PyDictValues *
+copy_values(PyDictValues *values)
+{
+ PyDictValues *newvalues = new_values(values->capacity);
+ if (newvalues == NULL) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ newvalues->size = values->size;
+ uint8_t *values_order = get_insertion_order_array(values);
+ uint8_t *new_values_order = get_insertion_order_array(newvalues);
+ memcpy(new_values_order, values_order, values->capacity);
+ for (int i = 0; i < values->capacity; i++) {
+ newvalues->values[i] = values->values[i];
+ }
+ assert(newvalues->embedded == 0);
+ return newvalues;
+}
+
static PyObject *
copy_lock_held(PyObject *o)
{
@@ -3833,26 +3866,23 @@ copy_lock_held(PyObject *o)
if (_PyDict_HasSplitTable(mp)) {
PyDictObject *split_copy;
- size_t size = shared_keys_usable_size(mp->ma_keys);
- PyDictValues *newvalues = new_values(size);
- if (newvalues == NULL)
+ PyDictValues *newvalues = copy_values(mp->ma_values);
+ if (newvalues == NULL) {
return PyErr_NoMemory();
+ }
split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type);
if (split_copy == NULL) {
free_values(newvalues, false);
return NULL;
}
- size_t prefix_size = ((uint8_t *)newvalues)[-1];
- memcpy(((char *)newvalues)-prefix_size, ((char
*)mp->ma_values)-prefix_size, prefix_size-1);
+ for (size_t i = 0; i < newvalues->capacity; i++) {
+ Py_XINCREF(newvalues->values[i]);
+ }
split_copy->ma_values = newvalues;
split_copy->ma_keys = mp->ma_keys;
split_copy->ma_used = mp->ma_used;
split_copy->ma_version_tag = DICT_NEXT_VERSION(interp);
dictkeys_incref(mp->ma_keys);
- for (size_t i = 0; i < size; i++) {
- PyObject *value = mp->ma_values->values[i];
- split_copy->ma_values->values[i] = Py_XNewRef(value);
- }
if (_PyObject_GC_IS_TRACKED(mp))
_PyObject_GC_TRACK(split_copy);
return (PyObject *)split_copy;
@@ -4406,8 +4436,10 @@ dict_traverse(PyObject *op, visitproc visit, void *arg)
if (DK_IS_UNICODE(keys)) {
if (_PyDict_HasSplitTable(mp)) {
- for (i = 0; i < n; i++) {
- Py_VISIT(mp->ma_values->values[i]);
+ if (!mp->ma_values->embedded) {
+ for (i = 0; i < n; i++) {
+ Py_VISIT(mp->ma_values->values[i]);
+ }
}
}
else {
@@ -5296,12 +5328,6 @@ acquire_key_value(PyObject **key_loc, PyObject *value,
PyObject **value_loc,
return 0;
}
-static Py_ssize_t
-load_values_used_size(PyDictValues *values)
-{
- return (Py_ssize_t)_Py_atomic_load_uint8(&DICT_VALUES_USED_SIZE(values));
-}
-
static int
dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self,
PyObject **out_key, PyObject **out_value)
@@ -5330,7 +5356,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject
*self,
goto concurrent_modification;
}
- Py_ssize_t used = load_values_used_size(values);
+ Py_ssize_t used = (Py_ssize_t)_Py_atomic_load_uint8(&values->size);
if (i >= used) {
goto fail;
}
@@ -6539,15 +6565,15 @@ _PyDict_NewKeysForClass(void)
return keys;
}
-#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys)
-
-int
+void
_PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp)
{
assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
+ assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictKeysObject *keys = CACHED_KEYS(tp);
assert(keys != NULL);
+ OBJECT_STAT_INC(inline_values);
#ifdef Py_GIL_DISABLED
Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable);
if (usable > 1) {
@@ -6563,49 +6589,19 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject
*tp)
}
#endif
size_t size = shared_keys_usable_size(keys);
- PyDictValues *values = new_values(size);
- if (values == NULL) {
- PyErr_NoMemory();
- return -1;
- }
- assert(((uint8_t *)values)[-1] >= (size + 2));
- ((uint8_t *)values)[-2] = 0;
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ assert(size < 256);
+ values->capacity = (uint8_t)size;
+ values->size = 0;
+ values->embedded = 1;
+ values->valid = 1;
for (size_t i = 0; i < size; i++) {
values->values[i] = NULL;
}
- _PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values);
- return 0;
-}
-
-int
-_PyObject_InitializeDict(PyObject *obj)
-{
- PyInterpreterState *interp = _PyInterpreterState_GET();
- PyTypeObject *tp = Py_TYPE(obj);
- if (tp->tp_dictoffset == 0) {
- return 0;
- }
- if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
- OBJECT_STAT_INC(new_values);
- return _PyObject_InitInlineValues(obj, tp);
- }
- PyObject *dict;
- if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) {
- dictkeys_incref(CACHED_KEYS(tp));
- dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
- }
- else {
- dict = PyDict_New();
- }
- if (dict == NULL) {
- return -1;
- }
- PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
- *dictptr = dict;
- return 0;
+ _PyObject_ManagedDictPointer(obj)->dict = NULL;
}
-static PyObject *
+static PyDictObject *
make_dict_from_instance_attributes(PyInterpreterState *interp,
PyDictKeysObject *keys, PyDictValues
*values)
{
@@ -6620,56 +6616,24 @@ make_dict_from_instance_attributes(PyInterpreterState
*interp,
track += _PyObject_GC_MAY_BE_TRACKED(val);
}
}
- PyObject *res = new_dict(interp, keys, values, used, 0);
+ PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used,
0);
if (track && res) {
_PyObject_GC_TRACK(res);
}
return res;
}
-PyObject *
-_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values)
+PyDictObject *
+_PyObject_MakeDictFromInstanceAttributes(PyObject *obj)
{
+ PyDictValues *values = _PyObject_InlineValues(obj);
PyInterpreterState *interp = _PyInterpreterState_GET();
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
OBJECT_STAT_INC(dict_materialized_on_request);
return make_dict_from_instance_attributes(interp, keys, values);
}
-// Return true if the dict was dematerialized, false otherwise.
-bool
-_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv)
-{
- assert(_PyObject_DictOrValuesPointer(obj) == dorv);
- assert(!_PyDictOrValues_IsValues(*dorv));
- PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv);
- if (dict == NULL) {
- return false;
- }
- // It's likely that this dict still shares its keys (if it was materialized
- // on request and not heavily modified):
- if (!PyDict_CheckExact(dict)) {
- return false;
- }
- assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE));
- if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) ||
- !has_unique_reference((PyObject *)dict))
- {
- return false;
- }
- ensure_shared_on_resize(dict);
- assert(dict->ma_values);
- // We have an opportunity to do something *really* cool: dematerialize it!
- _PyDictKeys_DecRef(dict->ma_keys);
- _PyDictOrValues_SetValues(dorv, dict->ma_values);
- OBJECT_STAT_INC(dict_dematerialized);
- // Don't try this at home, kids:
- dict->ma_keys = NULL;
- dict->ma_values = NULL;
- Py_DECREF(dict);
- return true;
-}
int
_PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
@@ -6679,7 +6643,7 @@ _PyObject_StoreInstanceAttribute(PyObject *obj,
PyDictValues *values,
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
assert(keys != NULL);
assert(values != NULL);
- assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
Py_ssize_t ix = DKIX_EMPTY;
if (PyUnicode_CheckExact(name)) {
Py_hash_t hash = unicode_get_hash(name);
@@ -6717,18 +6681,21 @@ _PyObject_StoreInstanceAttribute(PyObject *obj,
PyDictValues *values,
}
#endif
}
+ PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict;
if (ix == DKIX_EMPTY) {
- PyObject *dict = make_dict_from_instance_attributes(
- interp, keys, values);
if (dict == NULL) {
- return -1;
+ dict = make_dict_from_instance_attributes(
+ interp, keys, values);
+ if (dict == NULL) {
+ return -1;
+ }
+ _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict;
}
- _PyObject_DictOrValuesPointer(obj)->dict = dict;
if (value == NULL) {
- return PyDict_DelItem(dict, name);
+ return PyDict_DelItem((PyObject *)dict, name);
}
else {
- return PyDict_SetItem(dict, name, value);
+ return PyDict_SetItem((PyObject *)dict, name, value);
}
}
PyObject *old_value = values->values[ix];
@@ -6741,10 +6708,18 @@ _PyObject_StoreInstanceAttribute(PyObject *obj,
PyDictValues *values,
return -1;
}
_PyDictValues_AddToInsertionOrder(values, ix);
+ if (dict) {
+ assert(dict->ma_values == values);
+ dict->ma_used++;
+ }
}
else {
if (value == NULL) {
delete_index_from_values(values, ix);
+ if (dict) {
+ assert(dict->ma_values == values);
+ dict->ma_used--;
+ }
}
Py_DECREF(old_value);
}
@@ -6760,9 +6735,9 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj)
{
PyTypeObject *tp = Py_TYPE(obj);
CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
+ PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
+ if (_PyManagedDictPointer_IsValues(*managed_dict)) {
+ PyDictValues *values = _PyManagedDictPointer_GetValues(*managed_dict);
int size = ((uint8_t *)values)[-2];
int count = 0;
PyDictKeysObject *keys = CACHED_KEYS(tp);
@@ -6774,8 +6749,8 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj)
CHECK(size == count);
}
else {
- if (dorv_ptr->dict != NULL) {
- CHECK(PyDict_Check(dorv_ptr->dict));
+ if (managed_dict->dict != NULL) {
+ CHECK(PyDict_Check(managed_dict->dict));
}
}
return 1;
@@ -6804,23 +6779,27 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj)
if (tp->tp_dictoffset == 0) {
return 1;
}
- PyObject *dict;
- if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(dorv)) {
+ PyDictObject *dict;
+ if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (values->valid) {
PyDictKeysObject *keys = CACHED_KEYS(tp);
for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
- if (_PyDictOrValues_GetValues(dorv)->values[i] != NULL) {
+ if (values->values[i] != NULL) {
return 0;
}
}
return 1;
}
- dict = _PyDictOrValues_GetDict(dorv);
+ dict = _PyObject_ManagedDictPointer(obj)->dict;
+ }
+ else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+ PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
+ dict = managed_dict->dict;
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
- dict = *dictptr;
+ dict = (PyDictObject *)*dictptr;
}
if (dict == NULL) {
return 1;
@@ -6828,23 +6807,6 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj)
return ((PyDictObject *)dict)->ma_used == 0;
}
-void
-_PyObject_FreeInstanceAttributes(PyObject *self)
-{
- PyTypeObject *tp = Py_TYPE(self);
- assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self);
- if (!_PyDictOrValues_IsValues(dorv)) {
- return;
- }
- PyDictValues *values = _PyDictOrValues_GetValues(dorv);
- PyDictKeysObject *keys = CACHED_KEYS(tp);
- for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
- Py_XDECREF(values->values[i]);
- }
- free_values(values, IS_DICT_SHARED((PyDictObject*)self));
-}
-
int
PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
{
@@ -6852,74 +6814,101 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc
visit, void *arg)
if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
return 0;
}
- assert(tp->tp_dictoffset);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(dorv)) {
- PyDictValues *values = _PyDictOrValues_GetValues(dorv);
- PyDictKeysObject *keys = CACHED_KEYS(tp);
- for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
- Py_VISIT(values->values[i]);
+ if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (values->valid) {
+ for (Py_ssize_t i = 0; i < values->capacity; i++) {
+ Py_VISIT(values->values[i]);
+ }
+ return 0;
}
}
- else {
- PyObject *dict = _PyDictOrValues_GetDict(dorv);
- Py_VISIT(dict);
- }
+ Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict);
return 0;
}
void
PyObject_ClearManagedDict(PyObject *obj)
{
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ assert(_PyObject_InlineValuesConsistencyCheck(obj));
PyTypeObject *tp = Py_TYPE(obj);
- if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
- return;
- }
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
- PyDictKeysObject *keys = CACHED_KEYS(tp);
- for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
- Py_CLEAR(values->values[i]);
- }
- dorv_ptr->dict = NULL;
- free_values(values, IS_DICT_SHARED((PyDictObject*)obj));
- }
- else {
- PyObject *dict = dorv_ptr->dict;
+ if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict;
if (dict) {
- dorv_ptr->dict = NULL;
+ _PyDict_DetachFromObject(dict, obj);
+ _PyObject_ManagedDictPointer(obj)->dict = NULL;
Py_DECREF(dict);
}
+ else {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (values->valid) {
+ for (Py_ssize_t i = 0; i < values->capacity; i++) {
+ Py_CLEAR(values->values[i]);
+ }
+ values->valid = 0;
+ }
+ }
+ }
+ else {
+ Py_CLEAR(_PyObject_ManagedDictPointer(obj)->dict);
+ }
+ assert(_PyObject_InlineValuesConsistencyCheck(obj));
+}
+
+int
+_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj)
+{
+ assert(_PyObject_ManagedDictPointer(obj)->dict == mp);
+ assert(_PyObject_InlineValuesConsistencyCheck(obj));
+ if (mp->ma_values == NULL || mp->ma_values != _PyObject_InlineValues(obj))
{
+ return 0;
+ }
+ assert(mp->ma_values->embedded == 1);
+ assert(mp->ma_values->valid == 1);
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ Py_BEGIN_CRITICAL_SECTION(mp);
+ mp->ma_values = copy_values(mp->ma_values);
+ _PyObject_InlineValues(obj)->valid = 0;
+ Py_END_CRITICAL_SECTION();
+ if (mp->ma_values == NULL) {
+ return -1;
}
+ assert(_PyObject_InlineValuesConsistencyCheck(obj));
+ ASSERT_CONSISTENT(mp);
+ return 0;
}
PyObject *
PyObject_GenericGetDict(PyObject *obj, void *context)
{
- PyObject *dict;
PyInterpreterState *interp = _PyInterpreterState_GET();
PyTypeObject *tp = Py_TYPE(obj);
if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) {
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
+ PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
+ PyDictObject *dict = managed_dict->dict;
+ if (dict == NULL &&
+ (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
+ _PyObject_InlineValues(obj)->valid
+ ) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
OBJECT_STAT_INC(dict_materialized_on_request);
dict = make_dict_from_instance_attributes(
interp, CACHED_KEYS(tp), values);
if (dict != NULL) {
- dorv_ptr->dict = dict;
+ managed_dict->dict = (PyDictObject *)dict;
}
}
else {
- dict = _PyDictOrValues_GetDict(*dorv_ptr);
+ dict = managed_dict->dict;
if (dict == NULL) {
dictkeys_incref(CACHED_KEYS(tp));
OBJECT_STAT_INC(dict_materialized_on_request);
- dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
- dorv_ptr->dict = dict;
+ dict = (PyDictObject *)new_dict_with_shared_keys(interp,
CACHED_KEYS(tp));
+ managed_dict->dict = (PyDictObject *)dict;
}
}
+ return Py_XNewRef((PyObject *)dict);
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
@@ -6928,7 +6917,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context)
"This object has no __dict__");
return NULL;
}
- dict = *dictptr;
+ PyObject *dict = *dictptr;
if (dict == NULL) {
PyTypeObject *tp = Py_TYPE(obj);
if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) &&
CACHED_KEYS(tp)) {
@@ -6940,8 +6929,8 @@ PyObject_GenericGetDict(PyObject *obj, void *context)
*dictptr = dict = PyDict_New();
}
}
+ return Py_XNewRef(dict);
}
- return Py_XNewRef(dict);
}
int
@@ -6958,7 +6947,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject
**dictptr,
assert(dictptr != NULL);
dict = *dictptr;
if (dict == NULL) {
- assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT));
+ assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES));
dictkeys_incref(cached);
dict = new_dict_with_shared_keys(interp, cached);
if (dict == NULL)
@@ -7118,3 +7107,24 @@ _PyDict_SendEvent(int watcher_bits,
watcher_bits >>= 1;
}
}
+
+#ifndef NDEBUG
+static int
+_PyObject_InlineValuesConsistencyCheck(PyObject *obj)
+{
+ if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) {
+ return 1;
+ }
+ assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ PyDictObject *dict = (PyDictObject
*)_PyObject_ManagedDictPointer(obj)->dict;
+ if (dict == NULL) {
+ return 1;
+ }
+ if (dict->ma_values == _PyObject_InlineValues(obj) ||
+ _PyObject_InlineValues(obj)->valid == 0) {
+ return 1;
+ }
+ assert(0);
+ return 0;
+}
+#endif
diff --git a/Objects/object.c b/Objects/object.c
index b4f0fd4d7db941..60642d899bcafa 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1396,16 +1396,16 @@ _PyObject_GetDictPtr(PyObject *obj)
if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
return _PyObject_ComputedDictPointer(obj);
}
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj,
_PyDictOrValues_GetValues(*dorv_ptr));
+ PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
+ if (managed_dict->dict == NULL && Py_TYPE(obj)->tp_flags &
Py_TPFLAGS_INLINE_VALUES) {
+ PyDictObject *dict = (PyDictObject
*)_PyObject_MakeDictFromInstanceAttributes(obj);
if (dict == NULL) {
PyErr_Clear();
return NULL;
}
- dorv_ptr->dict = dict;
+ managed_dict->dict = dict;
}
- return &dorv_ptr->dict;
+ return (PyObject **)&managed_dict->dict;
}
PyObject *
@@ -1474,21 +1474,19 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name,
PyObject **method)
}
}
PyObject *dict;
- if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
- PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
- PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name);
- if (attr != NULL) {
- *method = attr;
- Py_XDECREF(descr);
- return 0;
- }
- dict = NULL;
- }
- else {
- dict = dorv_ptr->dict;
+ if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
_PyObject_InlineValues(obj)->valid) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name);
+ if (attr != NULL) {
+ *method = attr;
+ Py_XDECREF(descr);
+ return 0;
}
+ dict = NULL;
+ }
+ else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+ PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
+ dict = (PyObject *)managed_dict->dict;
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
@@ -1581,29 +1579,27 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj,
PyObject *name,
}
}
if (dict == NULL) {
- if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
- PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
- if (PyUnicode_CheckExact(name)) {
- res = _PyObject_GetInstanceAttribute(obj, values, name);
- if (res != NULL) {
- goto done;
- }
- }
- else {
- dict = _PyObject_MakeDictFromInstanceAttributes(obj,
values);
- if (dict == NULL) {
- res = NULL;
- goto done;
- }
- dorv_ptr->dict = dict;
+ if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
_PyObject_InlineValues(obj)->valid) {
+ PyDictValues *values = _PyObject_InlineValues(obj);
+ if (PyUnicode_CheckExact(name)) {
+ res = _PyObject_GetInstanceAttribute(obj, values, name);
+ if (res != NULL) {
+ goto done;
}
}
else {
- dict = _PyDictOrValues_GetDict(*dorv_ptr);
+ dict = (PyObject
*)_PyObject_MakeDictFromInstanceAttributes(obj);
+ if (dict == NULL) {
+ res = NULL;
+ goto done;
+ }
+ _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict;
}
}
+ else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+ PyManagedDictPointer* managed_dict =
_PyObject_ManagedDictPointer(obj);
+ dict = (PyObject *)managed_dict->dict;
+ }
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
if (dictptr) {
@@ -1697,22 +1693,14 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj,
PyObject *name,
if (dict == NULL) {
PyObject **dictptr;
- if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- res = _PyObject_StoreInstanceAttribute(
- obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value);
- goto error_check;
- }
- dictptr = &dorv_ptr->dict;
- if (*dictptr == NULL) {
- if (_PyObject_InitInlineValues(obj, tp) < 0) {
- goto done;
- }
- res = _PyObject_StoreInstanceAttribute(
- obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value);
- goto error_check;
- }
+ if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
_PyObject_InlineValues(obj)->valid) {
+ res = _PyObject_StoreInstanceAttribute(
+ obj, _PyObject_InlineValues(obj), name, value);
+ goto error_check;
+ }
+ else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+ PyManagedDictPointer *managed_dict =
_PyObject_ManagedDictPointer(obj);
+ dictptr = (PyObject **)&managed_dict->dict;
}
else {
dictptr = _PyObject_ComputedDictPointer(obj);
@@ -1783,9 +1771,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value,
void *context)
{
PyObject **dictptr = _PyObject_GetDictPtr(obj);
if (dictptr == NULL) {
- if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) &&
- _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(obj)))
- {
+ if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) &&
+ _PyObject_ManagedDictPointer(obj)->dict == NULL
+ ) {
/* Was unable to convert to dict */
PyErr_NoMemory();
}
diff --git a/Objects/object_layout.md b/Objects/object_layout.md
index 4f379bed8d77e2..352409425ee802 100644
--- a/Objects/object_layout.md
+++ b/Objects/object_layout.md
@@ -16,7 +16,45 @@ Since the introduction of the cycle GC, there has also been
a pre-header.
Before 3.11, this pre-header was two words in size.
It should be considered opaque to all code except the cycle GC.
-## 3.11 pre-header
+### 3.13
+
+In 3.13, the values array is embedded into the object, so there is no
+need for a values pointer (it is just a fixed offset into the object).
+So the pre-header is these two fields:
+
+* weakreflist
+* dict_pointer
+
+If the object has no physical dictionary, then the ``dict_pointer``
+is set to `NULL`.
+
+
+<details>
+<summary> 3.12 </summary>
+
+### 3.12
+
+In 3.12, the pointer to the list of weak references is added to the
+pre-header. In order to make space for it, the ``dict`` and ``values``
+pointers are combined into a single tagged pointer:
+
+* weakreflist
+* dict_or_values
+
+If the object has no physical dictionary, then the ``dict_or_values``
+has its low bit set to one, and points to the values array.
+If the object has a physical dictionary, then the ``dict_or_values``
+has its low bit set to zero, and points to the dictionary.
+
+The untagged form is chosen for the dictionary pointer, rather than
+the values pointer, to enable the (legacy) C-API function
+`_PyObject_GetDictPtr(PyObject *obj)` to work.
+</details>
+
+<details>
+<summary> 3.11 </summary>
+
+### 3.11
In 3.11 the pre-header was extended to include pointers to the VM managed
``__dict__``.
The reason for moving the ``__dict__`` to the pre-header is that it allows
@@ -33,27 +71,49 @@ The values pointer refers to the ``PyDictValues`` array
which holds the
values of the objects's attributes.
Should the dictionary be needed, then ``values`` is set to ``NULL``
and the ``dict`` field points to the dictionary.
+</details>
-## 3.12 pre-header
+## Layout of a "normal" Python object
-In 3.12, the pointer to the list of weak references is added to the
-pre-header. In order to make space for it, the ``dict`` and ``values``
-pointers are combined into a single tagged pointer:
+A "normal" Python object is one that doesn't inherit from a builtin
+class, doesn't have slots.
+
+### 3.13
+
+In 3.13 the values are embedded into the object, as follows:
* weakreflist
* dict_or_values
+* GC 1
+* GC 2
+* ob_refcnt
+* ob_type
+* Inlined values:
+ * Flags
+ * values 0
+ * values 1
+ * ...
+ * Insertion order bytes
-If the object has no physical dictionary, then the ``dict_or_values``
-has its low bit set to one, and points to the values array.
-If the object has a physical dictionary, then the ``dict_or_values``
-has its low bit set to zero, and points to the dictionary.
+This has all the advantages of the layout used in 3.12, plus:
+* Access to values is even faster as there is one less load
+* Fast access is mostly maintained when the `__dict__` is materialized
-The untagged form is chosen for the dictionary pointer, rather than
-the values pointer, to enable the (legacy) C-API function
-`_PyObject_GetDictPtr(PyObject *obj)` to work.
+
+
+For objects with opaque parts defined by a C extension,
+the layout is much the same as for 3.12
+
-## Layout of a "normal" Python object in 3.12:
+
+<details>
+<summary> 3.12 </summary>
+
+### 3.12:
+
+In 3.12, the header and pre-header form the entire object for "normal"
+Python objects:
* weakreflist
* dict_or_values
@@ -62,9 +122,6 @@ the values pointer, to enable the (legacy) C-API function
* ob_refcnt
* ob_type
-For a "normal" Python object, one that doesn't inherit from a builtin
-class or have slots, the header and pre-header form the entire object.
-
There are several advantages to this layout:
@@ -79,4 +136,6 @@ The full layout object, with an opaque part defined by a C
extension,
and `__slots__` looks like this:
+</details>
+
diff --git a/Objects/object_layout_312.gv b/Objects/object_layout_312.gv
index c0068d78568524..731a25332b3ace 100644
--- a/Objects/object_layout_312.gv
+++ b/Objects/object_layout_312.gv
@@ -20,6 +20,7 @@ digraph ideal {
shape = none
label = <<table border="0" cellspacing="0">
<tr><td><b>values</b></td></tr>
+ <tr><td border="1">Insertion order</td></tr>
<tr><td port="0" border="1">values[0]</td></tr>
<tr><td border="1">values[1]</td></tr>
<tr><td border="1">...</td></tr>
diff --git a/Objects/object_layout_312.png b/Objects/object_layout_312.png
index 396dab183b3e9b..a63d095ea0b19e 100644
Binary files a/Objects/object_layout_312.png and
b/Objects/object_layout_312.png differ
diff --git a/Objects/object_layout_313.gv b/Objects/object_layout_313.gv
new file mode 100644
index 00000000000000..d85352876f273b
--- /dev/null
+++ b/Objects/object_layout_313.gv
@@ -0,0 +1,45 @@
+digraph ideal {
+
+ rankdir = "LR"
+
+
+ object [
+ shape = none
+ label = <<table border="0" cellspacing="0">
+ <tr><td><b>object</b></td></tr>
+ <tr><td port="w" border="1">weakrefs</td></tr>
+ <tr><td port="dv" border="1">dict pointer</td></tr>
+ <tr><td border="1" >GC info 0</td></tr>
+ <tr><td border="1" >GC info 1</td></tr>
+ <tr><td port="r" border="1" >refcount</td></tr>
+ <tr><td port="h" border="1" >__class__</td></tr>
+ <tr><td border="1">values flags</td></tr>
+ <tr><td port="0" border="1">values[0]</td></tr>
+ <tr><td border="1">values[1]</td></tr>
+ <tr><td border="1">...</td></tr>
+ <tr><td border="1">Insertion order</td></tr>
+ </table>>
+
+ ]
+
+ class [
+ shape = none
+ label = <<table border="0" cellspacing="0">
+ <tr><td><b>class</b></td></tr>
+ <tr><td port="head" bgcolor="lightgreen"
border="1">...</td></tr>
+ <tr><td border="1"
bgcolor="lightgreen">dict_offset</td></tr>
+ <tr><td border="1" bgcolor="lightgreen">...</td></tr>
+ <tr><td port="k" border="1"
bgcolor="lightgreen">cached_keys</td></tr>
+ </table>>
+ ]
+
+ keys [label = "dictionary keys"; fillcolor="lightgreen"; style="filled"]
+ NULL [ label = " NULL"; shape="plain"]
+ object:w -> NULL
+ object:h -> class:head
+ object:dv -> NULL
+ class:k -> keys
+
+ oop [ label = "pointer"; shape="plain"]
+ oop -> object:r
+}
diff --git a/Objects/object_layout_313.png b/Objects/object_layout_313.png
new file mode 100644
index 00000000000000..f7059c286c84e6
Binary files /dev/null and b/Objects/object_layout_313.png differ
diff --git a/Objects/object_layout_full_313.gv
b/Objects/object_layout_full_313.gv
new file mode 100644
index 00000000000000..551312d51a7012
--- /dev/null
+++ b/Objects/object_layout_full_313.gv
@@ -0,0 +1,25 @@
+digraph ideal {
+
+ rankdir = "LR"
+
+
+ object [
+ shape = none
+ label = <<table border="0" cellspacing="0">
+ <tr><td><b>object</b></td></tr>
+ <tr><td port="w" border="1">weakrefs</td></tr>
+ <tr><td port="dv" border="1">dict pointer</td></tr>
+ <tr><td border="1" >GC info 0</td></tr>
+ <tr><td border="1" >GC info 1</td></tr>
+ <tr><td port="r" border="1" >refcount</td></tr>
+ <tr><td port="h" border="1" >__class__</td></tr>
+ <tr><td border="1">opaque (extension) data </td></tr>
+ <tr><td border="1">...</td></tr>
+ <tr><td border="1">__slot__ 0</td></tr>
+ <tr><td border="1">...</td></tr>
+ </table>>
+ ]
+
+ oop [ label = "pointer"; shape="plain"]
+ oop -> object:r
+}
diff --git a/Objects/object_layout_full_313.png
b/Objects/object_layout_full_313.png
new file mode 100644
index 00000000000000..7352ec69e5d8db
Binary files /dev/null and b/Objects/object_layout_full_313.png differ
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 2ef79fbf17b329..6df9986b82e77c 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1861,7 +1861,7 @@ type_call(PyObject *self, PyObject *args, PyObject *kwds)
PyObject *
_PyType_NewManagedObject(PyTypeObject *type)
{
- assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+ assert(type->tp_flags & Py_TPFLAGS_INLINE_VALUES);
assert(_PyType_IS_GC(type));
assert(type->tp_new == PyBaseObject_Type.tp_new);
assert(type->tp_alloc == PyType_GenericAlloc);
@@ -1870,11 +1870,6 @@ _PyType_NewManagedObject(PyTypeObject *type)
if (obj == NULL) {
return PyErr_NoMemory();
}
- _PyObject_DictOrValuesPointer(obj)->dict = NULL;
- if (_PyObject_InitInlineValues(obj, type)) {
- Py_DECREF(obj);
- return NULL;
- }
return obj;
}
@@ -1888,9 +1883,13 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t
nitems)
* flag to indicate when that is safe) it does not seem worth the memory
* savings. An example type that doesn't need the +1 is a subclass of
* tuple. See GH-100659 and GH-81381. */
- const size_t size = _PyObject_VAR_SIZE(type, nitems+1);
+ size_t size = _PyObject_VAR_SIZE(type, nitems+1);
const size_t presize = _PyType_PreHeaderSize(type);
+ if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ assert(type->tp_itemsize == 0);
+ size += _PyInlineValuesSize(type);
+ }
char *alloc = _PyObject_MallocWithType(type, size + presize);
if (alloc == NULL) {
return PyErr_NoMemory();
@@ -1911,6 +1910,9 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t
nitems)
else {
_PyObject_InitVar((PyVarObject *)obj, type, nitems);
}
+ if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ _PyObject_InitInlineValues(obj, type);
+ }
return obj;
}
@@ -2060,6 +2062,10 @@ subtype_clear(PyObject *self)
if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
PyObject_ClearManagedDict(self);
}
+ else {
+ assert((base->tp_flags & Py_TPFLAGS_INLINE_VALUES) ==
+ (type->tp_flags & Py_TPFLAGS_INLINE_VALUES));
+ }
}
else if (type->tp_dictoffset != base->tp_dictoffset) {
PyObject **dictptr = _PyObject_ComputedDictPointer(self);
@@ -2210,14 +2216,7 @@ subtype_dealloc(PyObject *self)
/* If we added a dict, DECREF it, or free inline values. */
if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
- PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(self);
- if (_PyDictOrValues_IsValues(*dorv_ptr)) {
- _PyObject_FreeInstanceAttributes(self);
- }
- else {
- Py_XDECREF(_PyDictOrValues_GetDict(*dorv_ptr));
- }
- dorv_ptr->values = NULL;
+ PyObject_ClearManagedDict(self);
}
else if (type->tp_dictoffset && !base->tp_dictoffset) {
PyObject **dictptr = _PyObject_ComputedDictPointer(self);
@@ -3161,19 +3160,26 @@ subtype_setdict(PyObject *obj, PyObject *value, void
*context)
return func(descr, obj, value);
}
/* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted.
*/
- dictptr = _PyObject_GetDictPtr(obj);
- if (dictptr == NULL) {
- PyErr_SetString(PyExc_AttributeError,
- "This object has no __dict__");
- return -1;
- }
if (value != NULL && !PyDict_Check(value)) {
PyErr_Format(PyExc_TypeError,
"__dict__ must be set to a dictionary, "
"not a '%.200s'", Py_TYPE(value)->tp_name);
return -1;
}
- Py_XSETREF(*dictptr, Py_XNewRef(value));
+ if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+ PyObject_ClearManagedDict(obj);
+ _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject
*)Py_XNewRef(value);
+ }
+ else {
+ dictptr = _PyObject_ComputedDictPointer(obj);
+ if (dictptr == NULL) {
+ PyErr_SetString(PyExc_AttributeError,
+ "This object has no __dict__");
+ return -1;
+ }
+ Py_CLEAR(*dictptr);
+ *dictptr = Py_XNewRef(value);
+ }
return 0;
}
@@ -5849,10 +5855,6 @@ object_new(PyTypeObject *type, PyObject *args, PyObject
*kwds)
if (obj == NULL) {
return NULL;
}
- if (_PyObject_InitializeDict(obj)) {
- Py_DECREF(obj);
- return NULL;
- }
return obj;
}
@@ -6036,6 +6038,11 @@ compatible_for_assignment(PyTypeObject* oldto,
PyTypeObject* newto, const char*
!same_slots_added(newbase, oldbase))) {
goto differs;
}
+ if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) !=
+ ((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES)))
+ {
+ goto differs;
+ }
/* The above does not check for the preheader */
if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) ==
((newto->tp_flags & Py_TPFLAGS_PREHEADER)))
@@ -6137,14 +6144,18 @@ object_set_class(PyObject *self, PyObject *value, void
*closure)
if (compatible_for_assignment(oldto, newto, "__class__")) {
/* Changing the class will change the implicit dict keys,
* so we must materialize the dictionary first. */
- assert((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == (newto->tp_flags &
Py_TPFLAGS_PREHEADER));
- _PyObject_GetDictPtr(self);
- if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT &&
- _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(self)))
- {
- /* Was unable to convert to dict */
- PyErr_NoMemory();
- return -1;
+ if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+ PyDictObject *dict = _PyObject_ManagedDictPointer(self)->dict;
+ if (dict == NULL) {
+ dict = (PyDictObject
*)_PyObject_MakeDictFromInstanceAttributes(self);
+ if (dict == NULL) {
+ return -1;
+ }
+ _PyObject_ManagedDictPointer(self)->dict = dict;
+ }
+ if (_PyDict_DetachFromObject(dict, self)) {
+ return -1;
+ }
}
if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) {
Py_INCREF(newto);
@@ -7774,6 +7785,9 @@ type_ready_managed_dict(PyTypeObject *type)
return -1;
}
}
+ if (type->tp_itemsize == 0 && type->tp_basicsize == sizeof(PyObject)) {
+ type->tp_flags |= Py_TPFLAGS_INLINE_VALUES;
+ }
return 0;
}
@@ -7901,6 +7915,8 @@ PyType_Ready(PyTypeObject *type)
/* Historically, all static types were immutable. See bpo-43908 */
if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE;
+ /* Static types must be immortal */
+ _Py_SetImmortalUntracked((PyObject *)type);
}
int res;
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index bfb378c4a41500..ce208aac9c7953 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -1897,14 +1897,12 @@ dummy_func(
op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_dictoffset < 0);
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv));
+ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(!_PyObject_InlineValues(owner)->valid);
}
split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if
(oparg & 1))) {
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- attr = _PyDictOrValues_GetValues(dorv)->values[index];
+ attr = _PyObject_InlineValues(owner)->values[index];
DEOPT_IF(attr == NULL);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
@@ -1947,16 +1945,15 @@ dummy_func(
op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(_PyDictOrValues_IsValues(dorv));
- PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
+ PyManagedDictPointer *managed_dict =
_PyObject_ManagedDictPointer(owner);
+ PyDictObject *dict = managed_dict->dict;
DEOPT_IF(dict == NULL);
assert(PyDict_CheckExact((PyObject *)dict));
}
op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1)))
{
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
+ PyManagedDictPointer *managed_dict =
_PyObject_ManagedDictPointer(owner);
+ PyDictObject *dict = managed_dict->dict;
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
@@ -2070,16 +2067,17 @@ dummy_func(
DISPATCH_INLINED(new_frame);
}
- op(_GUARD_DORV_VALUES, (owner -- owner)) {
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(!_PyDictOrValues_IsValues(dorv));
+ op(_GUARD_DORV_NO_DICT, (owner -- owner)) {
+ assert(Py_TYPE(owner)->tp_dictoffset < 0);
+ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict);
+ DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0);
}
op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) {
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
STAT_INC(STORE_ATTR, hit);
- PyDictValues *values = _PyDictOrValues_GetValues(dorv);
+ assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
+ PyDictValues *values = _PyObject_InlineValues(owner);
PyObject *old_value = values->values[index];
values->values[index] = value;
if (old_value == NULL) {
@@ -2094,7 +2092,7 @@ dummy_func(
macro(STORE_ATTR_INSTANCE_VALUE) =
unused/1 +
_GUARD_TYPE_VERSION +
- _GUARD_DORV_VALUES +
+ _GUARD_DORV_NO_DICT +
_STORE_ATTR_INSTANCE_VALUE;
inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value,
owner --)) {
@@ -2102,9 +2100,8 @@ dummy_func(
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(_PyDictOrValues_IsValues(dorv));
- PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
+ PyManagedDictPointer *managed_dict =
_PyObject_ManagedDictPointer(owner);
+ PyDictObject *dict = managed_dict->dict;
DEOPT_IF(dict == NULL);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
@@ -2898,9 +2895,8 @@ dummy_func(
}
op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) {
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv));
+ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(!_PyObject_InlineValues(owner)->valid);
}
op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) {
@@ -2972,10 +2968,9 @@ dummy_func(
unused/2 +
_LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
- op(_CHECK_ATTR_METHOD_LAZY_DICT, (owner -- owner)) {
- Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset;
- assert(dictoffset > 0);
- PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
+ op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) {
+ char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
+ PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL);
}
@@ -2993,7 +2988,7 @@ dummy_func(
unused/1 +
_GUARD_TYPE_VERSION +
_CHECK_ATTR_METHOD_LAZY_DICT +
- unused/2 +
+ unused/1 +
_LOAD_ATTR_METHOD_LAZY_DICT;
inst(INSTRUMENTED_CALL, (unused/3 -- )) {
@@ -3294,6 +3289,7 @@ dummy_func(
DEOPT_IF(!PyType_Check(callable));
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version));
+ assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable;
PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
PyCodeObject *code = (PyCodeObject *)init->func_code;
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index ce0dc235c54fcf..82f2171f1ede83 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -1753,9 +1753,8 @@
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_dictoffset < 0);
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
- if (!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET();
+ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET();
break;
}
@@ -1766,8 +1765,7 @@
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- attr = _PyDictOrValues_GetValues(dorv)->values[index];
+ attr = _PyObject_InlineValues(owner)->values[index];
if (attr == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
@@ -1784,8 +1782,7 @@
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- attr = _PyDictOrValues_GetValues(dorv)->values[index];
+ attr = _PyObject_InlineValues(owner)->values[index];
if (attr == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
@@ -1837,9 +1834,8 @@
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- if (_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET();
- PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
+ PyManagedDictPointer *managed_dict =
_PyObject_ManagedDictPointer(owner);
+ PyDictObject *dict = managed_dict->dict;
if (dict == NULL) JUMP_TO_JUMP_TARGET();
assert(PyDict_CheckExact((PyObject *)dict));
break;
@@ -1852,8 +1848,8 @@
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
uint16_t hint = (uint16_t)CURRENT_OPERAND();
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
+ PyManagedDictPointer *managed_dict =
_PyObject_ManagedDictPointer(owner);
+ PyDictObject *dict = managed_dict->dict;
if (hint >= (size_t)dict->ma_keys->dk_nentries)
JUMP_TO_JUMP_TARGET();
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
@@ -1967,12 +1963,13 @@
/* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for
tier 2 because it uses the 'this_instr' variable */
- case _GUARD_DORV_VALUES: {
+ case _GUARD_DORV_NO_DICT: {
PyObject *owner;
owner = stack_pointer[-1];
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- if (!_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET();
+ assert(Py_TYPE(owner)->tp_dictoffset < 0);
+ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ if (_PyObject_ManagedDictPointer(owner)->dict)
JUMP_TO_JUMP_TARGET();
+ if (_PyObject_InlineValues(owner)->valid == 0)
JUMP_TO_JUMP_TARGET();
break;
}
@@ -1982,9 +1979,9 @@
owner = stack_pointer[-1];
value = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND();
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
STAT_INC(STORE_ATTR, hit);
- PyDictValues *values = _PyDictOrValues_GetValues(dorv);
+ assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
+ PyDictValues *values = _PyObject_InlineValues(owner);
PyObject *old_value = values->values[index];
values->values[index] = value;
if (old_value == NULL) {
@@ -2568,9 +2565,8 @@
case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: {
PyObject *owner;
owner = stack_pointer[-1];
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
- if (!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET();
+ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET();
break;
}
@@ -2658,9 +2654,9 @@
case _CHECK_ATTR_METHOD_LAZY_DICT: {
PyObject *owner;
owner = stack_pointer[-1];
- Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset;
- assert(dictoffset > 0);
- PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
+ uint16_t dictoffset = (uint16_t)CURRENT_OPERAND();
+ char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
+ PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
if (dict != NULL) JUMP_TO_JUMP_TARGET();
break;
diff --git a/Python/gc.c b/Python/gc.c
index a37c1b144e57e9..a48738835fface 100644
--- a/Python/gc.c
+++ b/Python/gc.c
@@ -2031,11 +2031,16 @@ gc_alloc(PyTypeObject *tp, size_t basicsize, size_t
presize)
return op;
}
+
PyObject *
_PyObject_GC_New(PyTypeObject *tp)
{
size_t presize = _PyType_PreHeaderSize(tp);
- PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize);
+ size_t size = _PyObject_SIZE(tp);
+ if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) {
+ size += _PyInlineValuesSize(tp);
+ }
+ PyObject *op = gc_alloc(tp, size, presize);
if (op == NULL) {
return NULL;
}
diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c
index 4524382e4f689f..7e4137a8e342b1 100644
--- a/Python/gc_free_threading.c
+++ b/Python/gc_free_threading.c
@@ -1639,7 +1639,11 @@ PyObject *
_PyObject_GC_New(PyTypeObject *tp)
{
size_t presize = _PyType_PreHeaderSize(tp);
- PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize);
+ size_t size = _PyObject_SIZE(tp);
+ if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) {
+ size += _PyInlineValuesSize(tp);
+ }
+ PyObject *op = gc_alloc(tp, size, presize);
if (op == NULL) {
return NULL;
}
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index e8e2397b11cd48..6ee794a05b51d4 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -870,6 +870,7 @@
DEOPT_IF(!PyType_Check(callable), CALL);
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version),
CALL);
+ assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable;
PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
PyCodeObject *code = (PyCodeObject *)init->func_code;
@@ -3680,15 +3681,13 @@
// _CHECK_MANAGED_OBJECT_HAS_VALUES
{
assert(Py_TYPE(owner)->tp_dictoffset < 0);
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
+ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
}
// _LOAD_ATTR_INSTANCE_VALUE
{
uint16_t index = read_u16(&this_instr[4].cache);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- attr = _PyDictOrValues_GetValues(dorv)->values[index];
+ attr = _PyObject_InlineValues(owner)->values[index];
DEOPT_IF(attr == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
@@ -3721,13 +3720,13 @@
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
- Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset;
- assert(dictoffset > 0);
- PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
+ uint16_t dictoffset = read_u16(&this_instr[4].cache);
+ char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
+ PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
}
- /* Skip 2 cache entries */
+ /* Skip 1 cache entry */
// _LOAD_ATTR_METHOD_LAZY_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
@@ -3798,9 +3797,8 @@
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
+ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
@@ -3914,9 +3912,8 @@
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
+ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
@@ -4026,17 +4023,16 @@
// _CHECK_ATTR_WITH_HINT
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR);
- PyDictObject *dict = (PyDictObject
*)_PyDictOrValues_GetDict(dorv);
+ PyManagedDictPointer *managed_dict =
_PyObject_ManagedDictPointer(owner);
+ PyDictObject *dict = managed_dict->dict;
DEOPT_IF(dict == NULL, LOAD_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
}
// _LOAD_ATTR_WITH_HINT
{
uint16_t hint = read_u16(&this_instr[4].cache);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- PyDictObject *dict = (PyDictObject
*)_PyDictOrValues_GetDict(dorv);
+ PyManagedDictPointer *managed_dict =
_PyObject_ManagedDictPointer(owner);
+ PyDictObject *dict = managed_dict->dict;
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries,
LOAD_ATTR);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
@@ -5315,19 +5311,20 @@
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
- // _GUARD_DORV_VALUES
+ // _GUARD_DORV_NO_DICT
{
- assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR);
+ assert(Py_TYPE(owner)->tp_dictoffset < 0);
+ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict,
STORE_ATTR);
+ DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0,
STORE_ATTR);
}
// _STORE_ATTR_INSTANCE_VALUE
value = stack_pointer[-2];
{
uint16_t index = read_u16(&this_instr[4].cache);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
STAT_INC(STORE_ATTR, hit);
- PyDictValues *values = _PyDictOrValues_GetValues(dorv);
+ assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
+ PyDictValues *values = _PyObject_InlineValues(owner);
PyObject *old_value = values->values[index];
values->values[index] = value;
if (old_value == NULL) {
@@ -5389,9 +5386,8 @@
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
- PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
- DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR);
- PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
+ PyManagedDictPointer *managed_dict =
_PyObject_ManagedDictPointer(owner);
+ PyDictObject *dict = managed_dict->dict;
DEOPT_IF(dict == NULL, STORE_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index df73cc091dea26..b4a1da8aec14af 100644
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -1104,7 +1104,7 @@
/* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for
tier 2 */
- case _GUARD_DORV_VALUES: {
+ case _GUARD_DORV_NO_DICT: {
break;
}
diff --git a/Python/specialize.c b/Python/specialize.c
index c1edf8842faf68..f1e32d05af7707 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -188,7 +188,7 @@ print_object_stats(FILE *out, ObjectStats *stats)
fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n",
stats->allocations4k);
fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n",
stats->allocations_big);
fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
- fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
+ fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values);
fprintf(out, "Object interpreter increfs: %" PRIu64 "\n",
stats->interpreter_increfs);
fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n",
stats->interpreter_decrefs);
fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
@@ -197,7 +197,6 @@ print_object_stats(FILE *out, ObjectStats *stats)
fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n",
stats->dict_materialized_new_key);
fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n",
stats->dict_materialized_too_big);
fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n",
stats->dict_materialized_str_subclass);
- fprintf(out, "Object dematerialize dict: %" PRIu64 "\n",
stats->dict_dematerialized);
fprintf(out, "Object method cache hits: %" PRIu64 "\n",
stats->type_cache_hits);
fprintf(out, "Object method cache misses: %" PRIu64 "\n",
stats->type_cache_misses);
fprintf(out, "Object method cache collisions: %" PRIu64 "\n",
stats->type_cache_collisions);
@@ -479,12 +478,11 @@ _PyCode_Quicken(PyCodeObject *code)
#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18
#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19
#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20
-
#define SPEC_FAIL_ATTR_SHADOWED 21
#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22
#define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23
#define SPEC_FAIL_ATTR_OBJECT_SLOT 24
-#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25
+
#define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26
#define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27
#define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28
@@ -558,6 +556,7 @@ _PyCode_Quicken(PyCodeObject *code)
#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29
#define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30
#define SPEC_FAIL_CALL_METACLASS 31
+#define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32
/* COMPARE_OP */
#define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12
@@ -829,11 +828,7 @@ specialize_dict_access(
return 0;
}
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
- PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
- if (_PyDictOrValues_IsValues(*dorv) ||
- _PyObject_MakeInstanceAttributesFromDict(owner, dorv))
- {
- // Virtual dictionary
+ if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES &&
_PyObject_InlineValues(owner)->valid) {
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(PyUnicode_CheckExact(name));
Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
@@ -850,7 +845,8 @@ specialize_dict_access(
instr->op.code = values_op;
}
else {
- PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv);
+ PyManagedDictPointer *managed_dict =
_PyObject_ManagedDictPointer(owner);
+ PyDictObject *dict = managed_dict->dict;
if (dict == NULL || !PyDict_CheckExact(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
return 0;
@@ -1258,15 +1254,8 @@ PyObject *descr, DescriptorClassification kind, bool
is_method)
assert(descr != NULL);
assert((is_method && kind == METHOD) || (!is_method && kind ==
NON_DESCRIPTOR));
- if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
- PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
+ if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictKeysObject *keys = ((PyHeapTypeObject
*)owner_cls)->ht_cached_keys;
- if (!_PyDictOrValues_IsValues(*dorv) &&
- !_PyObject_MakeInstanceAttributesFromDict(owner, dorv))
- {
- SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT);
- return 0;
- }
Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
if (index != DKIX_EMPTY) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED);
@@ -1282,10 +1271,16 @@ PyObject *descr, DescriptorClassification kind, bool
is_method)
instr->op.code = is_method ? LOAD_ATTR_METHOD_WITH_VALUES :
LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES;
}
else {
- Py_ssize_t dictoffset = owner_cls->tp_dictoffset;
- if (dictoffset < 0 || dictoffset > INT16_MAX) {
- SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
- return 0;
+ Py_ssize_t dictoffset;
+ if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+ dictoffset = MANAGED_DICT_OFFSET;
+ }
+ else {
+ dictoffset = owner_cls->tp_dictoffset;
+ if (dictoffset < 0 || dictoffset > INT16_MAX +
MANAGED_DICT_OFFSET) {
+ SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
+ return 0;
+ }
}
if (dictoffset == 0) {
instr->op.code = is_method ? LOAD_ATTR_METHOD_NO_DICT :
LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
@@ -1296,8 +1291,12 @@ PyObject *descr, DescriptorClassification kind, bool
is_method)
SPECIALIZATION_FAIL(LOAD_ATTR,
SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
return 0;
}
- assert(owner_cls->tp_dictoffset > 0);
- assert(owner_cls->tp_dictoffset <= INT16_MAX);
+ /* Cache entries must be unsigned values, so we offset the
+ * dictoffset by MANAGED_DICT_OFFSET.
+ * We do the reverese offset in LOAD_ATTR_METHOD_LAZY_DICT */
+ dictoffset -= MANAGED_DICT_OFFSET;
+ assert(((uint16_t)dictoffset) == dictoffset);
+ cache->dict_offset = (uint16_t)dictoffset;
instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT;
}
else {
@@ -1729,8 +1728,8 @@ get_init_for_simple_managed_python_class(PyTypeObject *tp)
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN);
return NULL;
}
- if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
- SPECIALIZATION_FAIL(CALL, SPEC_FAIL_NO_DICT);
+ if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) {
+ SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES);
return NULL;
}
if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
diff --git a/Tools/cases_generator/analyzer.py
b/Tools/cases_generator/analyzer.py
index ddafcf99ca1e37..4261378d459107 100644
--- a/Tools/cases_generator/analyzer.py
+++ b/Tools/cases_generator/analyzer.py
@@ -359,11 +359,10 @@ def has_error_without_pop(op: parser.InstDef) -> bool:
NON_ESCAPING_FUNCTIONS = (
"Py_INCREF",
- "_PyDictOrValues_IsValues",
- "_PyObject_DictOrValuesPointer",
- "_PyDictOrValues_GetValues",
+ "_PyManagedDictPointer_IsValues",
+ "_PyObject_ManagedDictPointer",
+ "_PyObject_InlineValues",
"_PyDictValues_AddToInsertionOrder",
- "_PyObject_MakeInstanceAttributesFromDict",
"Py_DECREF",
"_Py_DECREF_SPECIALIZED",
"DECREF_INPUTS_AND_REUSE_FLOAT",
diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py
index 656667ac93970c..74165acd831131 100755
--- a/Tools/gdb/libpython.py
+++ b/Tools/gdb/libpython.py
@@ -66,10 +66,12 @@ def _type_unsigned_short_ptr():
def _type_unsigned_int_ptr():
return gdb.lookup_type('unsigned int').pointer()
-
def _sizeof_void_p():
return gdb.lookup_type('void').pointer().sizeof
+def _sizeof_pyobject():
+ return gdb.lookup_type('PyObject').sizeof
+
def _managed_dict_offset():
# See pycore_object.h
pyobj = gdb.lookup_type("PyObject")
@@ -79,6 +81,7 @@ def _managed_dict_offset():
return -3 * _sizeof_void_p()
+Py_TPFLAGS_INLINE_VALUES = (1 << 2)
Py_TPFLAGS_MANAGED_DICT = (1 << 4)
Py_TPFLAGS_HEAPTYPE = (1 << 9)
Py_TPFLAGS_LONG_SUBCLASS = (1 << 24)
@@ -493,11 +496,12 @@ def get_keys_values(self):
has_values = int_from_int(typeobj.field('tp_flags')) &
Py_TPFLAGS_MANAGED_DICT
if not has_values:
return None
- ptr = self._gdbval.cast(_type_char_ptr()) + _managed_dict_offset()
- char_ptr = ptr.cast(_type_char_ptr().pointer()).dereference()
- if (int(char_ptr) & 1) == 0:
+ obj_ptr = self._gdbval.cast(_type_char_ptr())
+ dict_ptr_ptr = obj_ptr + _managed_dict_offset()
+ dict_ptr = dict_ptr_ptr.cast(_type_char_ptr().pointer()).dereference()
+ if int(dict_ptr):
return None
- char_ptr += 1
+ char_ptr = obj_ptr + _sizeof_pyobject()
values_ptr = char_ptr.cast(gdb.lookup_type("PyDictValues").pointer())
values = values_ptr['values']
return PyKeysValuesPair(self.get_cached_keys(), values)
diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index 8dc590b4b89a88..f7ed98ff6045ab 100644
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -394,7 +394,7 @@ def get_call_stats(self) -> dict[str, int]:
return result
def get_object_stats(self) -> dict[str, tuple[int, int]]:
- total_materializations = self._data.get("Object new values", 0)
+ total_materializations = self._data.get("Object inline values", 0)
total_allocations = self._data.get("Object allocations", 0) +
self._data.get(
"Object allocations from freelist", 0
)
@@ -1094,8 +1094,7 @@ def calc_object_stats_table(stats: Stats) -> Rows:
Below, "allocations" means "allocations that are not from a freelist".
Total allocations = "Allocations from freelist" + "Allocations".
- "New values" is the number of values arrays created for objects with
- managed dicts.
+ "Inline values" is the number of values arrays inlined into objects.
The cache hit/miss numbers are for the MRO cache, split into dunder and
other names.
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/python-checkins.python.org/
Member address: [email protected]
