https://github.com/python/cpython/commit/c0af5c024b57d216fc3db41cb0e39a683c327bbd
commit: c0af5c024b57d216fc3db41cb0e39a683c327bbd
branch: main
author: Dino Viehland <[email protected]>
committer: DinoV <[email protected]>
date: 2026-04-16T09:44:26-07:00
summary:
gh-146031: Allow keeping specialization enabled when specifying eval frame
function (#146032)
Allow keeping specialization enabled when specifying eval frame function
files:
A
Misc/NEWS.d/next/Core_and_Builtins/2026-03-16-17-29-22.gh-issue-146031.6nyB7C.rst
M Doc/c-api/subinterpreters.rst
M Include/cpython/pystate.h
M Include/internal/pycore_interp_structs.h
M Lib/test/test_capi/test_misc.py
M Modules/_testinternalcapi.c
M Modules/_testinternalcapi/interpreter.c
M Python/ceval_macros.h
M Python/pystate.c
M Python/specialize.c
diff --git a/Doc/c-api/subinterpreters.rst b/Doc/c-api/subinterpreters.rst
index 44e3fc96841aac..83c3fc3d801e9b 100644
--- a/Doc/c-api/subinterpreters.rst
+++ b/Doc/c-api/subinterpreters.rst
@@ -399,6 +399,27 @@ High-level APIs
.. versionadded:: 3.9
+.. c:function:: void
_PyInterpreterState_SetEvalFrameAllowSpecialization(PyInterpreterState *interp,
int allow_specialization)
+
+ Enables or disables specialization why a custom frame evaluator is in place.
+
+ If *allow_specialization* is non-zero, the adaptive specializer will
+ continue to specialize bytecodes even though a custom eval frame function
+ is set. When *allow_specialization* is zero, setting a custom eval frame
+ disables specialization. The standard interpreter loop will continue to
deopt
+ while a frame evaluation API is in place - the frame evaluation function
needs
+ to handle the specialized opcodes to take advantage of this.
+
+ .. versionadded:: 3.15
+
+.. c:function:: int
_PyInterpreterState_IsSpecializationEnabled(PyInterpreterState *interp)
+
+ Return non-zero if adaptive specialization is enabled for the interpreter.
+ Specialization is enabled when no custom eval frame function is set, or
+ when one is set with *allow_specialization* enabled.
+
+ .. versionadded:: 3.15
+
Low-level APIs
--------------
diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index 1c56ad5af8072f..0cb57679df331d 100644
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -319,3 +319,8 @@ PyAPI_FUNC(_PyFrameEvalFunction)
_PyInterpreterState_GetEvalFrameFunc(
PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc(
PyInterpreterState *interp,
_PyFrameEvalFunction eval_frame);
+PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameAllowSpecialization(
+ PyInterpreterState *interp,
+ int allow_specialization);
+PyAPI_FUNC(int) _PyInterpreterState_IsSpecializationEnabled(
+ PyInterpreterState *interp);
diff --git a/Include/internal/pycore_interp_structs.h
b/Include/internal/pycore_interp_structs.h
index c4b084642668a9..2bfb84da36cbc8 100644
--- a/Include/internal/pycore_interp_structs.h
+++ b/Include/internal/pycore_interp_structs.h
@@ -927,6 +927,7 @@ struct _is {
PyObject *builtins_copy;
// Initialized to _PyEval_EvalFrameDefault().
_PyFrameEvalFunction eval_frame;
+ int eval_frame_allow_specialization;
PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS];
// One bit is set for each non-NULL entry in func_watchers
diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py
index db06719919535f..4c16bbd4cb0acf 100644
--- a/Lib/test/test_capi/test_misc.py
+++ b/Lib/test/test_capi/test_misc.py
@@ -2870,6 +2870,88 @@ def func():
self.do_test(func, names)
+class Test_Pep523AllowSpecialization(unittest.TestCase):
+ """Tests for _PyInterpreterState_SetEvalFrameFunc with
+ allow_specialization=1."""
+
+ def test_is_specialization_enabled_default(self):
+ # With no custom eval frame, specialization should be enabled
+ self.assertTrue(_testinternalcapi.is_specialization_enabled())
+
+ def test_is_specialization_enabled_with_eval_frame(self):
+ # Setting eval frame with allow_specialization=0 disables
specialization
+ try:
+ _testinternalcapi.set_eval_frame_record([])
+ self.assertFalse(_testinternalcapi.is_specialization_enabled())
+ finally:
+ _testinternalcapi.set_eval_frame_default()
+
+ def test_is_specialization_enabled_after_restore(self):
+ # Restoring the default eval frame re-enables specialization
+ try:
+ _testinternalcapi.set_eval_frame_record([])
+ self.assertFalse(_testinternalcapi.is_specialization_enabled())
+ finally:
+ _testinternalcapi.set_eval_frame_default()
+ self.assertTrue(_testinternalcapi.is_specialization_enabled())
+
+ def test_is_specialization_enabled_with_allow(self):
+ # Setting eval frame with allow_specialization=1 keeps it enabled
+ try:
+ _testinternalcapi.set_eval_frame_interp([])
+ self.assertTrue(_testinternalcapi.is_specialization_enabled())
+ finally:
+ _testinternalcapi.set_eval_frame_default()
+
+ def test_allow_specialization_call(self):
+ def func():
+ pass
+
+ def func_outer():
+ func()
+
+ actual_calls = []
+ try:
+ _testinternalcapi.set_eval_frame_interp(
+ actual_calls)
+ for i in range(SUFFICIENT_TO_DEOPT_AND_SPECIALIZE * 2):
+ func_outer()
+ finally:
+ _testinternalcapi.set_eval_frame_default()
+
+ # With specialization enabled, calls to inner() will dispatch
+ # through the installed frame evaluator
+ self.assertEqual(actual_calls.count("func"), 0)
+
+ # But the normal interpreter loop still shouldn't be inlining things
+ self.assertNotEqual(actual_calls.count("func_outer"), 0)
+
+ def test_no_specialization_call(self):
+ # Without allow_specialization, ALL calls go through the eval frame.
+ # This is the existing PEP 523 behavior.
+ def inner(x=42):
+ pass
+ def func():
+ inner()
+
+ # Pre-specialize
+ for _ in range(SUFFICIENT_TO_DEOPT_AND_SPECIALIZE):
+ func()
+
+ actual_calls = []
+ try:
+ _testinternalcapi.set_eval_frame_record(actual_calls)
+ for _ in range(SUFFICIENT_TO_DEOPT_AND_SPECIALIZE):
+ func()
+ finally:
+ _testinternalcapi.set_eval_frame_default()
+
+ # Without allow_specialization, every call including inner() goes
+ # through the eval frame
+ expected = ["func", "inner"] * SUFFICIENT_TO_DEOPT_AND_SPECIALIZE
+ self.assertEqual(actual_calls, expected)
+
+
@unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED')
class TestPyThreadId(unittest.TestCase):
def test_py_thread_id(self):
diff --git
a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-16-17-29-22.gh-issue-146031.6nyB7C.rst
b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-16-17-29-22.gh-issue-146031.6nyB7C.rst
new file mode 100644
index 00000000000000..cabcf975e5aa89
--- /dev/null
+++
b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-16-17-29-22.gh-issue-146031.6nyB7C.rst
@@ -0,0 +1 @@
+The unstable API _PyInterpreterState_SetEvalFrameFunc has a companion function
_PyInterpreterState_SetEvalFrameAllowSpecialization to specify if
specialization should be allowed. When this option is set to 1 the specializer
will turn Python -> Python calls into specialized opcodes which the replacement
interpreter loop can choose to respect and perform inlined dispatch.
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index c00bad46a54907..deac8570fe3241 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -996,12 +996,51 @@ get_eval_frame_stats(PyObject *self, PyObject
*Py_UNUSED(args))
}
static PyObject *
-set_eval_frame_interp(PyObject *self, PyObject *Py_UNUSED(args))
+record_eval_interp(PyThreadState *tstate, struct _PyInterpreterFrame *f, int
exc)
{
- _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(),
Test_EvalFrame);
+ if (PyStackRef_FunctionCheck(f->f_funcobj)) {
+ PyFunctionObject *func = _PyFrame_GetFunction(f);
+ PyObject *module = _get_current_module();
+ assert(module != NULL);
+ module_state *state = get_module_state(module);
+ Py_DECREF(module);
+ int res = PyList_Append(state->record_list, func->func_name);
+ if (res < 0) {
+ return NULL;
+ }
+ }
+
+ return Test_EvalFrame(tstate, f, exc);
+}
+
+static PyObject *
+set_eval_frame_interp(PyObject *self, PyObject *args)
+{
+ if (PyTuple_GET_SIZE(args) == 1) {
+ module_state *state = get_module_state(self);
+ PyObject *list = PyTuple_GET_ITEM(args, 0);
+ if (!PyList_Check(list)) {
+ PyErr_SetString(PyExc_TypeError, "argument must be a list");
+ return NULL;
+ }
+ Py_XSETREF(state->record_list, Py_NewRef(list));
+ _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(),
record_eval_interp);
+
_PyInterpreterState_SetEvalFrameAllowSpecialization(_PyInterpreterState_GET(),
1);
+ } else {
+ _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(),
Test_EvalFrame);
+
_PyInterpreterState_SetEvalFrameAllowSpecialization(_PyInterpreterState_GET(),
1);
+ }
+
Py_RETURN_NONE;
}
+static PyObject *
+is_specialization_enabled(PyObject *self, PyObject *Py_UNUSED(args))
+{
+ return PyBool_FromLong(
+
_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET()));
+}
+
/*[clinic input]
_testinternalcapi.compiler_cleandoc -> object
@@ -2875,8 +2914,9 @@ static PyMethodDef module_functions[] = {
{"EncodeLocaleEx", encode_locale_ex, METH_VARARGS},
{"DecodeLocaleEx", decode_locale_ex, METH_VARARGS},
{"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL},
- {"set_eval_frame_interp", set_eval_frame_interp, METH_NOARGS, NULL},
+ {"set_eval_frame_interp", set_eval_frame_interp, METH_VARARGS, NULL},
{"set_eval_frame_record", set_eval_frame_record, METH_O, NULL},
+ {"is_specialization_enabled", is_specialization_enabled, METH_NOARGS,
NULL},
_TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF
_TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF
_TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF
diff --git a/Modules/_testinternalcapi/interpreter.c
b/Modules/_testinternalcapi/interpreter.c
index 2cd23fa3c58849..99dcd18393fb87 100644
--- a/Modules/_testinternalcapi/interpreter.c
+++ b/Modules/_testinternalcapi/interpreter.c
@@ -9,6 +9,9 @@
#include "../../Python/ceval_macros.h"
+#undef IS_PEP523_HOOKED
+#define IS_PEP523_HOOKED(tstate) (tstate->interp->eval_frame != NULL &&
!tstate->interp->eval_frame_allow_specialization)
+
int Test_EvalFrame_Resumes, Test_EvalFrame_Loads;
#ifdef _Py_TIER2
diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index 62e9d11aeb3af2..a7d63fd3b82211 100644
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -220,14 +220,14 @@ do { \
DISPATCH_GOTO_NON_TRACING(); \
}
-#define DISPATCH_INLINED(NEW_FRAME) \
- do { \
- assert(tstate->interp->eval_frame == NULL); \
- _PyFrame_SetStackPointer(frame, stack_pointer); \
- assert((NEW_FRAME)->previous == frame); \
- frame = tstate->current_frame = (NEW_FRAME); \
- CALL_STAT_INC(inlined_py_calls); \
- JUMP_TO_LABEL(start_frame); \
+#define DISPATCH_INLINED(NEW_FRAME) \
+ do { \
+ assert(!IS_PEP523_HOOKED(tstate)); \
+ _PyFrame_SetStackPointer(frame, stack_pointer); \
+ assert((NEW_FRAME)->previous == frame); \
+ frame = tstate->current_frame = (NEW_FRAME); \
+ CALL_STAT_INC(inlined_py_calls); \
+ JUMP_TO_LABEL(start_frame); \
} while (0)
/* Tuple access macros */
diff --git a/Python/pystate.c b/Python/pystate.c
index 3f539a4c2551ba..d6a26f3339b863 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -3026,9 +3026,32 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState
*interp,
RARE_EVENT_INC(set_eval_frame_func);
_PyEval_StopTheWorld(interp);
interp->eval_frame = eval_frame;
+ // reset when evaluator is reset
+ interp->eval_frame_allow_specialization = 0;
_PyEval_StartTheWorld(interp);
}
+void
+_PyInterpreterState_SetEvalFrameAllowSpecialization(PyInterpreterState *interp,
+ int allow_specialization)
+{
+ if (allow_specialization == interp->eval_frame_allow_specialization) {
+ return;
+ }
+ _Py_Executors_InvalidateAll(interp, 1);
+ RARE_EVENT_INC(set_eval_frame_func);
+ _PyEval_StopTheWorld(interp);
+ interp->eval_frame_allow_specialization = allow_specialization;
+ _PyEval_StartTheWorld(interp);
+}
+
+int
+_PyInterpreterState_IsSpecializationEnabled(PyInterpreterState *interp)
+{
+ return interp->eval_frame == NULL
+ || interp->eval_frame_allow_specialization;
+}
+
const PyConfig*
_PyInterpreterState_GetConfig(PyInterpreterState *interp)
diff --git a/Python/specialize.c b/Python/specialize.c
index cadd25314686d5..793bac58adf41a 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -838,7 +838,7 @@ do_specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject*
return -1;
}
/* Don't specialize if PEP 523 is active */
- if (_PyInterpreterState_GET()->eval_frame) {
+ if
(!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
return -1;
}
@@ -922,7 +922,7 @@ do_specialize_instance_load_attr(PyObject* owner,
_Py_CODEUNIT* instr, PyObject*
return -1;
}
/* Don't specialize if PEP 523 is active */
- if (_PyInterpreterState_GET()->eval_frame) {
+ if
(!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
return -1;
}
@@ -1740,7 +1740,7 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT
*instr, int nargs,
PyCodeObject *code = (PyCodeObject *)func->func_code;
int kind = function_kind(code);
/* Don't specialize if PEP 523 is active */
- if (_PyInterpreterState_GET()->eval_frame) {
+ if
(!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523);
return -1;
}
@@ -1783,7 +1783,7 @@ specialize_py_call_kw(PyFunctionObject *func,
_Py_CODEUNIT *instr, int nargs,
PyCodeObject *code = (PyCodeObject *)func->func_code;
int kind = function_kind(code);
/* Don't specialize if PEP 523 is active */
- if (_PyInterpreterState_GET()->eval_frame) {
+ if
(!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523);
return -1;
}
@@ -2046,7 +2046,7 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject
*rhs)
return SPEC_FAIL_WRONG_NUMBER_ARGUMENTS;
}
- if (_PyInterpreterState_GET()->eval_frame) {
+ if
(!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) {
/* Don't specialize if PEP 523 is active */
Py_DECREF(descriptor);
return SPEC_FAIL_OTHER;
@@ -2449,7 +2449,7 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef
rhs_st, _Py_CODEUNIT *in
PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type;
if (kind == SIMPLE_FUNCTION &&
fcode->co_argcount == 2 &&
- !_PyInterpreterState_GET()->eval_frame && /* Don't
specialize if PEP 523 is active */
+
_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET()) && /*
Don't specialize if PEP 523 is active */
_PyType_CacheGetItemForSpecialization(ht, descriptor,
(uint32_t)tp_version))
{
specialize(instr, BINARY_OP_SUBSCR_GETITEM);
@@ -2707,7 +2707,7 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef
null_or_index, _Py_CODEUNIT
instr[oparg + INLINE_CACHE_ENTRIES_FOR_ITER + 1].op.code ==
INSTRUMENTED_END_FOR
);
/* Don't specialize if PEP 523 is active */
- if (_PyInterpreterState_GET()->eval_frame) {
+ if
(!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) {
goto failure;
}
specialize(instr, FOR_ITER_GEN);
@@ -2750,7 +2750,7 @@ _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT
*instr)
PyTypeObject *tp = Py_TYPE(receiver);
if (tp == &PyGen_Type || tp == &PyCoro_Type) {
/* Don't specialize if PEP 523 is active */
- if (_PyInterpreterState_GET()->eval_frame) {
+ if
(!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) {
SPECIALIZATION_FAIL(SEND, SPEC_FAIL_OTHER);
goto failure;
}
@@ -2773,7 +2773,7 @@ _Py_Specialize_CallFunctionEx(_PyStackRef func_st,
_Py_CODEUNIT *instr)
if (Py_TYPE(func) == &PyFunction_Type &&
((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
- if (_PyInterpreterState_GET()->eval_frame) {
+ if
(!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) {
goto failure;
}
specialize(instr, CALL_EX_PY);
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
