https://github.com/python/cpython/commit/ccf1b0b1c18e6d00fb919bce107f2793bab0a471 commit: ccf1b0b1c18e6d00fb919bce107f2793bab0a471 branch: main author: Mark Shannon <m...@hotpy.org> committer: markshannon <m...@hotpy.org> date: 2025-04-29T18:00:35+01:00 summary:
GH-132508: Use tagged integers on the evaluation stack for the last instruction offset (GH-132545) files: A Misc/NEWS.d/next/Core_and_Builtins/2025-04-15-10-09-49.gh-issue-132508.zVe3iI.rst M Include/internal/pycore_interpframe.h M Include/internal/pycore_stackref.h M Objects/frameobject.c M Python/bytecodes.c M Python/ceval.c M Python/executor_cases.c.h M Python/gc.c M Python/gc_free_threading.c M Python/generated_cases.c.h M Python/pystate.c M Python/stackrefs.c M Tools/cases_generator/analyzer.py M Tools/jit/_stencils.py diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 1d373d55ab2de8..d3fd218b27eed7 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -18,6 +18,7 @@ extern "C" { ((int)((IF)->instr_ptr - _PyFrame_GetBytecode((IF)))) static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { + assert(!PyStackRef_IsNull(f->f_executable)); PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); assert(PyCode_Check(executable)); return (PyCodeObject *)executable; diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 5683b98470d3ea..a2acf311ff4c65 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -63,11 +63,13 @@ extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _P static const _PyStackRef PyStackRef_NULL = { .index = 0 }; -#define PyStackRef_None ((_PyStackRef){ .index = 1 } ) -#define PyStackRef_False ((_PyStackRef){ .index = 2 }) -#define PyStackRef_True ((_PyStackRef){ .index = 3 }) +// Use the first 3 even numbers for None, True and False. +// Odd numbers are reserved for (tagged) integers +#define PyStackRef_None ((_PyStackRef){ .index = 2 } ) +#define PyStackRef_False ((_PyStackRef){ .index = 4 }) +#define PyStackRef_True ((_PyStackRef){ .index = 6 }) -#define LAST_PREDEFINED_STACKREF_INDEX 3 +#define INITIAL_STACKREF_INDEX 8 static inline int PyStackRef_IsNull(_PyStackRef ref) @@ -96,6 +98,7 @@ PyStackRef_IsNone(_PyStackRef ref) static inline PyObject * _PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber) { + assert((ref.index & 1) == 0); _Py_stackref_record_borrow(ref, filename, linenumber); return _Py_stackref_get_object(ref); } @@ -132,31 +135,45 @@ _PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenu } #define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__) +static inline bool +PyStackRef_IsTaggedInt(_PyStackRef ref) +{ + return (ref.index & 1) == 1; +} + static inline void _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber) { + if (PyStackRef_IsTaggedInt(ref)) { + return; + } PyObject *obj = _Py_stackref_close(ref, filename, linenumber); Py_DECREF(obj); } #define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__) + static inline void _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber) { if (PyStackRef_IsNull(ref)) { return; } - PyObject *obj = _Py_stackref_close(ref, filename, linenumber); - Py_DECREF(obj); + _PyStackRef_CLOSE(ref, filename, linenumber); } #define PyStackRef_XCLOSE(REF) _PyStackRef_XCLOSE((REF), __FILE__, __LINE__) static inline _PyStackRef _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) { - PyObject *obj = _Py_stackref_get_object(ref); - Py_INCREF(obj); - return _Py_stackref_create(obj, filename, linenumber); + if (PyStackRef_IsTaggedInt(ref)) { + return ref; + } + else { + PyObject *obj = _Py_stackref_get_object(ref); + Py_INCREF(obj); + return _Py_stackref_create(obj, filename, linenumber); + } } #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) @@ -210,8 +227,40 @@ _PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linen extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b); +extern bool PyStackRef_IsTaggedInt(_PyStackRef ref); + +extern intptr_t PyStackRef_UntagInt(_PyStackRef ref); + +extern _PyStackRef PyStackRef_TagInt(intptr_t i); + +extern bool +PyStackRef_IsNullOrInt(_PyStackRef ref); + #else +#define Py_INT_TAG 3 + +static inline bool +PyStackRef_IsTaggedInt(_PyStackRef i) +{ + return (i.bits & Py_INT_TAG) == Py_INT_TAG; +} + +static inline _PyStackRef +PyStackRef_TagInt(intptr_t i) +{ + assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i); + return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) }; +} + +static inline intptr_t +PyStackRef_UntagInt(_PyStackRef i) +{ + assert((i.bits & Py_INT_TAG) == Py_INT_TAG); + intptr_t val = (intptr_t)i.bits; + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 2); +} + #ifdef Py_GIL_DISABLED @@ -232,6 +281,8 @@ static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; #define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True) #define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False) +#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref)) + static inline PyObject * PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) { @@ -451,6 +502,7 @@ PyStackRef_RefcountOnObject(_PyStackRef ref) static inline PyObject * PyStackRef_AsPyObjectBorrow(_PyStackRef ref) { + assert(!PyStackRef_IsTaggedInt(ref)); return BITS_TO_PTR_MASKED(ref); } @@ -587,6 +639,12 @@ PyStackRef_CLOSE(_PyStackRef ref) } #endif +static inline bool +PyStackRef_IsNullOrInt(_PyStackRef ref) +{ + return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref); +} + static inline void PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) { @@ -726,7 +784,7 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) // Like Py_VISIT but for _PyStackRef fields #define _Py_VISIT_STACKREF(ref) \ do { \ - if (!PyStackRef_IsNull(ref)) { \ + if (!PyStackRef_IsNullOrInt(ref)) { \ int vret = _PyGC_VisitStackRef(&(ref), visit, arg); \ if (vret) \ return vret; \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-15-10-09-49.gh-issue-132508.zVe3iI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-15-10-09-49.gh-issue-132508.zVe3iI.rst new file mode 100644 index 00000000000000..1f4fe1d5f4e2da --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-15-10-09-49.gh-issue-132508.zVe3iI.rst @@ -0,0 +1,3 @@ +Uses tagged integers on the evaluation stack to represent the instruction +offsets when reraising an exception. This avoids the need to box the integer +which could fail in low memory conditions. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index db4e4b7e1939de..7a62219c139ee5 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1813,15 +1813,16 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value) start_stack = pop_value(start_stack); } while (start_stack > best_stack) { + _PyStackRef popped = _PyFrame_StackPop(self->f_frame); if (top_of_stack(start_stack) == Except) { /* Pop exception stack as well as the evaluation stack */ - PyObject *exc = PyStackRef_AsPyObjectBorrow(_PyFrame_StackPop(self->f_frame)); + PyObject *exc = PyStackRef_AsPyObjectBorrow(popped); assert(PyExceptionInstance_Check(exc) || exc == Py_None); PyThreadState *tstate = _PyThreadState_GET(); Py_XSETREF(tstate->exc_info->exc_value, exc == Py_None ? NULL : exc); } else { - PyStackRef_XCLOSE(_PyFrame_StackPop(self->f_frame)); + PyStackRef_XCLOSE(popped); } start_stack = pop_value(start_stack); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4e2dec7c9852ee..6592bc57ed20a1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1384,16 +1384,7 @@ dummy_func( assert(oparg >= 0 && oparg <= 2); if (oparg) { - PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); - if (PyLong_Check(lasti)) { - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); - assert(!_PyErr_Occurred(tstate)); - } - else { - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - Py_DECREF(exc); - ERROR_NO_POP(); - } + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); } assert(exc && PyExceptionInstance_Check(exc)); _PyErr_SetRaisedException(tstate, exc); @@ -3472,7 +3463,7 @@ dummy_func( if (tb == NULL) { tb = Py_None; } - assert(PyStackRef_LongCheck(lasti)); + assert(PyStackRef_IsTaggedInt(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); @@ -5378,11 +5369,8 @@ dummy_func( } if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - goto exception_unwind; - } - _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); + _PyStackRef lasti = PyStackRef_TagInt(frame_lasti); + _PyFrame_StackPush(frame, lasti); } /* Make the raw exception data diff --git a/Python/ceval.c b/Python/ceval.c index fb72fd49811e2d..19a1c9529dd9aa 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -146,6 +146,10 @@ dump_item(_PyStackRef item) printf("<NULL>"); return; } + if (PyStackRef_IsTaggedInt(item)) { + printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item)); + return; + } PyObject *obj = PyStackRef_AsPyObjectBorrow(item); if (obj == NULL) { printf("<nil>"); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 09998090575e31..4e57906c84a154 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4470,7 +4470,7 @@ if (tb == NULL) { tb = Py_None; } - assert(PyStackRef_LongCheck(lasti)); + assert(PyStackRef_IsTaggedInt(lasti)); (void)lasti; PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); diff --git a/Python/gc.c b/Python/gc.c index dad088e09f872f..58224acff2cdd9 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -547,6 +547,7 @@ _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg) // This is a bit tricky! We want to ignore stackrefs with embedded // refcounts when computing the incoming references, but otherwise treat // them like normal. + assert(!PyStackRef_IsTaggedInt(*ref)); if (!PyStackRef_RefcountOnObject(*ref) && (visit == visit_decref)) { return 0; } @@ -560,7 +561,9 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) _PyStackRef *ref = _PyFrame_GetLocalsArray(frame); /* locals and stack */ for (; ref < frame->stackpointer; ref++) { - _Py_VISIT_STACKREF(*ref); + if (!PyStackRef_IsTaggedInt(*ref)) { + _Py_VISIT_STACKREF(*ref); + } } return 0; } @@ -1495,8 +1498,11 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b objects_marked += move_to_reachable(func, &reachable, visited_space); while (sp > locals) { sp--; + if (PyStackRef_IsNullOrInt(*sp)) { + continue; + } PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); - if (op == NULL || _Py_IsImmortal(op)) { + if (_Py_IsImmortal(op)) { continue; } if (_PyObject_IS_GC(op)) { diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index fa4cb56f01e800..d22307ae4ff74e 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -265,7 +265,7 @@ frame_disable_deferred_refcounting(_PyInterpreterFrame *frame) frame->f_funcobj = PyStackRef_AsStrongReference(frame->f_funcobj); for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) { - if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) { + if (!PyStackRef_IsNullOrInt(*ref) && PyStackRef_IsDeferred(*ref)) { *ref = PyStackRef_AsStrongReference(*ref); } } @@ -420,7 +420,7 @@ gc_visit_heaps(PyInterpreterState *interp, mi_block_visit_fun *visitor, static inline void gc_visit_stackref(_PyStackRef stackref) { - if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNull(stackref)) { + if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsNullOrInt(stackref)) { PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); if (_PyObject_GC_IS_TRACKED(obj) && !gc_is_frozen(obj)) { gc_add_refs(obj, 1); @@ -817,7 +817,7 @@ gc_abort_mark_alive(PyInterpreterState *interp, static int gc_visit_stackref_mark_alive(gc_mark_args_t *args, _PyStackRef stackref) { - if (!PyStackRef_IsNull(stackref)) { + if (!PyStackRef_IsNullOrInt(stackref)) { PyObject *op = PyStackRef_AsPyObjectBorrow(stackref); if (gc_mark_enqueue(op, args) < 0) { return -1; @@ -1706,6 +1706,7 @@ _PyGC_VisitStackRef(_PyStackRef *ref, visitproc visit, void *arg) // This is a bit tricky! We want to ignore deferred references when // computing the incoming references, but otherwise treat them like // regular references. + assert(!PyStackRef_IsTaggedInt(*ref)); if (!PyStackRef_IsDeferred(*ref) || (visit != visit_decref && visit != visit_decref_unreachable)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index fbd1f2349d0c4a..7d3e6c7cbc9eab 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10191,20 +10191,7 @@ PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); assert(oparg >= 0 && oparg <= 2); if (oparg) { - PyObject *lasti = PyStackRef_AsPyObjectBorrow(values[0]); - if (PyLong_Check(lasti)) { - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyLong_AsLong(lasti); - assert(!_PyErr_Occurred(tstate)); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int"); - Py_DECREF(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); } assert(exc && PyExceptionInstance_Check(exc)); stack_pointer += -1; @@ -12059,7 +12046,7 @@ if (tb == NULL) { tb = Py_None; } - assert(PyStackRef_LongCheck(lasti)); + assert(PyStackRef_IsTaggedInt(lasti)); (void)lasti; PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); @@ -12231,11 +12218,8 @@ JUMP_TO_LABEL(error); } if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); - PyObject *lasti = PyLong_FromLong(frame_lasti); - if (lasti == NULL) { - JUMP_TO_LABEL(exception_unwind); - } - _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(lasti)); + _PyStackRef lasti = PyStackRef_TagInt(frame_lasti); + _PyFrame_StackPush(frame, lasti); } PyObject *exc = _PyErr_GetRaisedException(tstate); _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); diff --git a/Python/pystate.c b/Python/pystate.c index 8c0bab3f2b48f8..b0c79ba9d3e645 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -684,7 +684,7 @@ init_interpreter(PyInterpreterState *interp, interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); } #if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) - interp->next_stackref = 1; + interp->next_stackref = INITIAL_STACKREF_INDEX; _Py_hashtable_allocator_t alloc = { .malloc = malloc, .free = free, diff --git a/Python/stackrefs.c b/Python/stackrefs.c index 1a2f66feb04c4d..979a6b1c62820a 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -70,7 +70,7 @@ _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber) } PyObject *obj; - if (ref.index <= LAST_PREDEFINED_STACKREF_INDEX) { + if (ref.index < INITIAL_STACKREF_INDEX) { if (ref.index == 0) { _Py_FatalErrorFormat(__func__, "Passing NULL to PyStackRef_CLOSE at %s:%d\n", filename, linenumber); } @@ -113,7 +113,8 @@ _Py_stackref_create(PyObject *obj, const char *filename, int linenumber) Py_FatalError("Cannot create a stackref for NULL"); } PyInterpreterState *interp = PyInterpreterState_Get(); - uint64_t new_id = interp->next_stackref++; + uint64_t new_id = interp->next_stackref; + interp->next_stackref = new_id + 2; TableEntry *entry = make_table_entry(obj, filename, linenumber); if (entry == NULL) { Py_FatalError("No memory left for stackref debug table"); @@ -127,7 +128,7 @@ _Py_stackref_create(PyObject *obj, const char *filename, int linenumber) void _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber) { - if (ref.index <= LAST_PREDEFINED_STACKREF_INDEX) { + if (ref.index < INITIAL_STACKREF_INDEX) { return; } PyInterpreterState *interp = PyInterpreterState_Get(); @@ -151,8 +152,7 @@ _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref) { - assert(interp->next_stackref >= ref.index); - interp->next_stackref = ref.index+1; + assert(ref.index < INITIAL_STACKREF_INDEX); TableEntry *entry = make_table_entry(obj, "builtin-object", 0); if (entry == NULL) { Py_FatalError("No memory left for stackref debug table"); @@ -197,4 +197,23 @@ _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char * _Py_DECREF_SPECIALIZED(obj, destruct); } +_PyStackRef PyStackRef_TagInt(intptr_t i) +{ + return (_PyStackRef){ .index = (i << 1) + 1 }; +} + +intptr_t +PyStackRef_UntagInt(_PyStackRef i) +{ + assert(PyStackRef_IsTaggedInt(i)); + intptr_t val = (intptr_t)i.index; + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 1); +} + +bool +PyStackRef_IsNullOrInt(_PyStackRef ref) +{ + return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref); +} + #endif diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index a217d7136a5401..dddbf2cf872e3d 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -678,6 +678,9 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "JUMP_TO_LABEL", "restart_backoff_counter", "_Py_ReachedRecursionLimit", + "PyStackRef_IsTaggedInt", + "PyStackRef_TagInt", + "PyStackRef_UntagInt", ) def check_escaping_calls(instr: parser.CodeDef, escapes: dict[SimpleStmt, EscapingCall]) -> None: diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 22d3014351df90..03b0ba647b0db7 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -293,6 +293,7 @@ def process_relocations( hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"} and hole.value is HoleValue.ZERO + and hole.symbol not in self.symbols ): hole.func = "patch_aarch64_trampoline" hole.need_state = True _______________________________________________ Python-checkins mailing list -- python-checkins@python.org To unsubscribe send an email to python-checkins-le...@python.org https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: arch...@mail-archive.com