https://github.com/python/cpython/commit/b4b73245d88c61b4c97dff98f846897d869c42ac
commit: b4b73245d88c61b4c97dff98f846897d869c42ac
branch: main
author: Hai Zhu <[email protected]>
committer: Fidget-Spinner <[email protected]>
date: 2026-01-17T15:52:16Z
summary:
gh-143421: Use new buffer to save optimized uops (GH-143682)
files:
M Include/internal/pycore_optimizer_types.h
M Include/internal/pycore_tstate.h
M Python/optimizer.c
M Python/optimizer_analysis.c
M Python/optimizer_bytecodes.c
M Python/optimizer_cases.c.h
M Tools/cases_generator/optimizer_generator.py
diff --git a/Include/internal/pycore_optimizer_types.h
b/Include/internal/pycore_optimizer_types.h
index 10828d1027dd09..6501ce869c1425 100644
--- a/Include/internal/pycore_optimizer_types.h
+++ b/Include/internal/pycore_optimizer_types.h
@@ -129,6 +129,8 @@ typedef struct _JitOptContext {
JitOptRef *n_consumed;
JitOptRef *limit;
JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE];
+ _PyUOpInstruction *out_buffer;
+ int out_len;
} JitOptContext;
diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h
index dc4444717b4e87..24a40416c2191b 100644
--- a/Include/internal/pycore_tstate.h
+++ b/Include/internal/pycore_tstate.h
@@ -60,6 +60,7 @@ typedef struct _PyJitTracerState {
_PyJitTracerTranslatorState translator_state;
JitOptContext opt_context;
_PyUOpInstruction code_buffer[UOP_MAX_TRACE_LENGTH];
+ _PyUOpInstruction out_buffer[UOP_MAX_TRACE_LENGTH];
} _PyJitTracerState;
#endif
diff --git a/Python/optimizer.c b/Python/optimizer.c
index 9892a9731e24f0..ab0ef3db4e4882 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -1535,6 +1535,7 @@ uop_optimize(
if (length <= 0) {
return length;
}
+ buffer = _tstate->jit_tracer_state->out_buffer;
}
assert(length < UOP_MAX_TRACE_LENGTH/2);
assert(length >= 1);
diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index cd3b79a2c13140..d635ebabf9007a 100644
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -176,6 +176,18 @@ incorrect_keys(PyObject *obj, uint32_t version)
#define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack))
#define STACK_SIZE() ((int)(ctx->frame->stack_len))
+static inline int
+is_terminator_uop(const _PyUOpInstruction *uop)
+{
+ int opcode = uop->opcode;
+ return (
+ opcode == _EXIT_TRACE ||
+ opcode == _JUMP_TO_TOP ||
+ opcode == _DYNAMIC_EXIT ||
+ opcode == _DEOPT
+ );
+}
+
#define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject
*)&_Py_InitCleanup))
#define GETLOCAL(idx) ((ctx->frame->locals[idx]))
@@ -185,6 +197,22 @@ incorrect_keys(PyObject *obj, uint32_t version)
(INST)->oparg = ARG; \
(INST)->operand0 = OPERAND;
+#define ADD_OP(OP, ARG, OPERAND) add_op(ctx, this_instr, (OP), (ARG),
(OPERAND))
+
+static inline void
+add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr,
+ uint16_t opcode, uint16_t oparg, uintptr_t operand0)
+{
+ _PyUOpInstruction *out = &ctx->out_buffer[ctx->out_len];
+ out->opcode = (opcode);
+ out->format = this_instr->format;
+ out->oparg = (oparg);
+ out->target = this_instr->target;
+ out->operand0 = (operand0);
+ out->operand1 = this_instr->operand1;
+ ctx->out_len++;
+}
+
/* Shortened forms for convenience, used in optimizer_bytecodes.c */
#define sym_is_not_null _Py_uop_sym_is_not_null
#define sym_is_const _Py_uop_sym_is_const
@@ -252,7 +280,7 @@ optimize_to_bool(
bool insert_mode)
{
if (sym_matches_type(value, &PyBool_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
*result_ptr = value;
return 1;
}
@@ -262,7 +290,7 @@ optimize_to_bool(
int opcode = insert_mode ?
_INSERT_1_LOAD_CONST_INLINE_BORROW :
_POP_TOP_LOAD_CONST_INLINE_BORROW;
- REPLACE_OP(this_instr, opcode, 0, (uintptr_t)load);
+ ADD_OP(opcode, 0, (uintptr_t)load);
*result_ptr = sym_new_const(ctx, load);
return 1;
}
@@ -270,9 +298,9 @@ optimize_to_bool(
}
static void
-eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit)
+eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool
exit)
{
- REPLACE_OP(this_instr, _POP_TOP, 0, 0);
+ ADD_OP(_POP_TOP, 0, 0);
if (exit) {
REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0);
this_instr[1].target = this_instr->target;
@@ -289,7 +317,7 @@ lookup_attr(JitOptContext *ctx, _PyBloomFilter
*dependencies, _PyUOpInstruction
PyObject *lookup = _PyType_Lookup(type, name);
if (lookup) {
int opcode = _Py_IsImmortal(lookup) ? immortal : mortal;
- REPLACE_OP(this_instr, opcode, 0, (uintptr_t)lookup);
+ ADD_OP(opcode, 0, (uintptr_t)lookup);
PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type);
_Py_BloomFilter_Add(dependencies, type);
return sym_new_const(ctx, lookup);
@@ -382,6 +410,8 @@ optimize_uops(
JitOptContext *ctx = &tstate->jit_tracer_state->opt_context;
uint32_t opcode = UINT16_MAX;
+ ctx->out_buffer = tstate->jit_tracer_state->out_buffer;
+
// Make sure that watchers are set up
PyInterpreterState *interp = _PyInterpreterState_GET();
if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) {
@@ -398,6 +428,8 @@ optimize_uops(
ctx->curr_frame_depth++;
ctx->frame = frame;
+ ctx->out_len = 0;
+
_PyUOpInstruction *this_instr = NULL;
JitOptRef *stack_pointer = ctx->frame->stack_pointer;
@@ -431,6 +463,10 @@ optimize_uops(
DPRINTF(1, "\nUnknown opcode in abstract interpreter\n");
Py_UNREACHABLE();
}
+ // If no ADD_OP was called during this iteration, copy the original
instruction
+ if (ctx->out_len == i) {
+ ctx->out_buffer[ctx->out_len++] = *this_instr;
+ }
assert(ctx->frame != NULL);
if (!CURRENT_FRAME_IS_INIT_SHIM()) {
DPRINTF(3, " stack_level %d\n", STACK_LEVEL());
@@ -459,7 +495,21 @@ optimize_uops(
/* Either reached the end or cannot optimize further, but there
* would be no benefit in retrying later */
_Py_uop_abstractcontext_fini(ctx);
- return trace_len;
+ // Check that the trace ends with a proper terminator
+ if (ctx->out_len > 0) {
+ _PyUOpInstruction *last_uop = &ctx->out_buffer[ctx->out_len - 1];
+ if (!is_terminator_uop(last_uop)) {
+ // Copy remaining uops from original trace until we find a
terminator
+ for (int i = ctx->out_len; i < trace_len; i++) {
+ ctx->out_buffer[ctx->out_len++] = trace[i];
+ if (is_terminator_uop(&trace[i])) {
+ break;
+ }
+ }
+ }
+ }
+
+ return ctx->out_len;
error:
DPRINTF(3, "\n");
@@ -631,7 +681,7 @@ _Py_uop_analyze_and_optimize(
assert(length > 0);
- length = remove_unneeded_uops(buffer, length);
+ length = remove_unneeded_uops(tstate->jit_tracer_state->out_buffer,
length);
assert(length > 0);
OPT_STAT_INC(optimizer_successes);
diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c
index 8fae031e11b4b0..876ba7c6de7482 100644
--- a/Python/optimizer_bytecodes.c
+++ b/Python/optimizer_bytecodes.c
@@ -48,7 +48,7 @@ optimize_to_bool(
bool insert_mode);
extern void
-eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit);
+eliminate_pop_guard(_PyUOpInstruction *this_instr, JitOptContext *ctx, bool
exit);
extern PyCodeObject *get_code(_PyUOpInstruction *op);
@@ -139,11 +139,11 @@ dummy_func(void) {
op(_GUARD_TOS_INT, (value -- value)) {
if (sym_is_compact_int(value)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
else {
if (sym_get_type(value) == &PyLong_Type) {
- REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0);
+ ADD_OP(_GUARD_TOS_OVERFLOWED, 0, 0);
}
sym_set_compact_int(value);
}
@@ -151,11 +151,11 @@ dummy_func(void) {
op(_GUARD_NOS_INT, (left, unused -- left, unused)) {
if (sym_is_compact_int(left)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
else {
if (sym_get_type(left) == &PyLong_Type) {
- REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0);
+ ADD_OP(_GUARD_NOS_OVERFLOWED, 0, 0);
}
sym_set_compact_int(left);
}
@@ -165,7 +165,7 @@ dummy_func(void) {
PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version);
if (type) {
if (type == sym_get_const(ctx, owner)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
else {
sym_set_const(owner, type);
@@ -176,7 +176,7 @@ dummy_func(void) {
op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) {
assert(type_version);
if (sym_matches_type_version(owner, type_version)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
} else {
// add watcher so that whenever the type changes we invalidate this
PyTypeObject *type = _PyType_LookupByVersion(type_version);
@@ -198,14 +198,14 @@ dummy_func(void) {
op(_GUARD_TOS_FLOAT, (value -- value)) {
if (sym_matches_type(value, &PyFloat_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(value, &PyFloat_Type);
}
op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) {
if (sym_matches_type(left, &PyFloat_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(left, &PyFloat_Type);
}
@@ -363,7 +363,7 @@ dummy_func(void) {
assert(index >= 0);
int tuple_length = sym_tuple_length(tuple_st);
if (tuple_length != -1 && index < tuple_length) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
}
}
@@ -438,7 +438,7 @@ dummy_func(void) {
op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) {
if (sym_matches_type(nos, &PyUnicode_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(nos, &PyUnicode_Type);
}
@@ -449,7 +449,7 @@ dummy_func(void) {
op(_GUARD_TOS_UNICODE, (value -- value)) {
if (sym_matches_type(value, &PyUnicode_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(value, &PyUnicode_Type);
}
@@ -560,7 +560,7 @@ dummy_func(void) {
op(_LOAD_CONST, (-- value)) {
PyCodeObject *co = get_current_code_object(ctx);
PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg);
- REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);
+ ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);
value = PyJitRef_Borrow(sym_new_const(ctx, val));
}
@@ -568,7 +568,7 @@ dummy_func(void) {
PyObject *val = PyLong_FromLong(oparg);
assert(val);
assert(_Py_IsImmortal(val));
- REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);
+ ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);
value = PyJitRef_Borrow(sym_new_const(ctx, val));
}
@@ -606,34 +606,34 @@ dummy_func(void) {
if (PyJitRef_IsBorrowed(value) ||
sym_is_immortal(PyJitRef_Unwrap(value)) ||
sym_is_null(value)) {
- REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0);
+ ADD_OP(_POP_TOP_NOP, 0, 0);
}
else if (typ == &PyLong_Type) {
- REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0);
+ ADD_OP(_POP_TOP_INT, 0, 0);
}
else if (typ == &PyFloat_Type) {
- REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0);
+ ADD_OP(_POP_TOP_FLOAT, 0, 0);
}
else if (typ == &PyUnicode_Type) {
- REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0);
+ ADD_OP(_POP_TOP_UNICODE, 0, 0);
}
}
op(_POP_TOP_INT, (value --)) {
if (PyJitRef_IsBorrowed(value)) {
- REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0);
+ ADD_OP(_POP_TOP_NOP, 0, 0);
}
}
op(_POP_TOP_FLOAT, (value --)) {
if (PyJitRef_IsBorrowed(value)) {
- REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0);
+ ADD_OP(_POP_TOP_NOP, 0, 0);
}
}
op(_POP_TOP_UNICODE, (value --)) {
if (PyJitRef_IsBorrowed(value)) {
- REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0);
+ ADD_OP(_POP_TOP_NOP, 0, 0);
}
}
@@ -686,11 +686,11 @@ dummy_func(void) {
op (_PUSH_NULL_CONDITIONAL, ( -- null[oparg & 1])) {
if (oparg & 1) {
- REPLACE_OP(this_instr, _PUSH_NULL, 0, 0);
+ ADD_OP(_PUSH_NULL, 0, 0);
null[0] = sym_new_null(ctx);
}
else {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
}
@@ -795,8 +795,8 @@ dummy_func(void) {
op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null,
unused[oparg] -- callable, self_or_null, unused[oparg])) {
if (sym_is_const(ctx, callable) && sym_matches_type(callable,
&PyFunction_Type)) {
assert(PyFunction_Check(sym_get_const(ctx, callable)));
- REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0,
func_version);
- this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable);
+ ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version);
+ ctx->out_buffer[ctx->out_len - 1].operand1 =
(uintptr_t)sym_get_const(ctx, callable);
}
sym_set_type(callable, &PyFunction_Type);
}
@@ -805,8 +805,8 @@ dummy_func(void) {
if (sym_is_const(ctx, callable) && sym_matches_type(callable,
&PyMethod_Type)) {
PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx,
callable);
assert(PyMethod_Check(method));
- REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0,
func_version);
- this_instr->operand1 = (uintptr_t)method->im_func;
+ ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version);
+ ctx->out_buffer[ctx->out_len - 1].operand1 =
(uintptr_t)method->im_func;
}
sym_set_type(callable, &PyMethod_Type);
}
@@ -818,7 +818,7 @@ dummy_func(void) {
PyFunctionObject *func = (PyFunctionObject
*)sym_get_const(ctx, callable);
PyCodeObject *co = (PyCodeObject *)func->func_code;
if (co->co_argcount == oparg + !sym_is_null(self_or_null)) {
- REPLACE_OP(this_instr, _NOP, 0 ,0);
+ ADD_OP(_NOP, 0 ,0);
}
}
}
@@ -1081,7 +1081,7 @@ dummy_func(void) {
op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) {
if (sym_matches_type(iter, &PyTuple_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(iter, &PyTuple_Type);
}
@@ -1094,7 +1094,7 @@ dummy_func(void) {
PyObject* type = (PyObject *)sym_get_type(arg);
if (type) {
res = sym_new_const(ctx, type);
- REPLACE_OP(this_instr, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0,
+ ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)type);
}
else {
@@ -1132,7 +1132,7 @@ dummy_func(void) {
out = Py_True;
}
sym_set_const(res, out);
- REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)out);
+ ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out);
}
}
@@ -1140,7 +1140,7 @@ dummy_func(void) {
if (sym_is_const(ctx, flag)) {
PyObject *value = sym_get_const(ctx, flag);
assert(value != NULL);
- eliminate_pop_guard(this_instr, value != Py_True);
+ eliminate_pop_guard(this_instr, ctx, value != Py_True);
}
sym_set_const(flag, Py_True);
}
@@ -1185,7 +1185,7 @@ dummy_func(void) {
if (sym_is_const(ctx, flag)) {
PyObject *value = sym_get_const(ctx, flag);
assert(value != NULL);
- eliminate_pop_guard(this_instr, value != Py_False);
+ eliminate_pop_guard(this_instr, ctx, value != Py_False);
}
sym_set_const(flag, Py_False);
}
@@ -1194,11 +1194,11 @@ dummy_func(void) {
if (sym_is_const(ctx, val)) {
PyObject *value = sym_get_const(ctx, val);
assert(value != NULL);
- eliminate_pop_guard(this_instr, !Py_IsNone(value));
+ eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value));
}
else if (sym_has_type(val)) {
assert(!sym_matches_type(val, &_PyNone_Type));
- eliminate_pop_guard(this_instr, true);
+ eliminate_pop_guard(this_instr, ctx, true);
}
sym_set_const(val, Py_None);
}
@@ -1207,11 +1207,11 @@ dummy_func(void) {
if (sym_is_const(ctx, val)) {
PyObject *value = sym_get_const(ctx, val);
assert(value != NULL);
- eliminate_pop_guard(this_instr, Py_IsNone(value));
+ eliminate_pop_guard(this_instr, ctx, Py_IsNone(value));
}
else if (sym_has_type(val)) {
assert(!sym_matches_type(val, &_PyNone_Type));
- eliminate_pop_guard(this_instr, false);
+ eliminate_pop_guard(this_instr, ctx, false);
}
}
@@ -1219,7 +1219,7 @@ dummy_func(void) {
/* Setting the eval frame function invalidates
* all executors, so no need to check dynamically */
if (_PyInterpreterState_GET()->eval_frame == NULL) {
- REPLACE_OP(this_instr, _NOP, 0 ,0);
+ ADD_OP(_NOP, 0 ,0);
}
}
@@ -1247,7 +1247,7 @@ dummy_func(void) {
}
op(_REPLACE_WITH_TRUE, (value -- res, v)) {
- REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)Py_True);
+ ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True);
res = sym_new_const(ctx, Py_True);
v = value;
}
@@ -1302,42 +1302,42 @@ dummy_func(void) {
op(_GUARD_TOS_LIST, (tos -- tos)) {
if (sym_matches_type(tos, &PyList_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(tos, &PyList_Type);
}
op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) {
if (sym_matches_type(nos, &PyList_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(nos, &PyList_Type);
}
op(_GUARD_TOS_TUPLE, (tos -- tos)) {
if (sym_matches_type(tos, &PyTuple_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(tos, &PyTuple_Type);
}
op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) {
if (sym_matches_type(nos, &PyTuple_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(nos, &PyTuple_Type);
}
op(_GUARD_TOS_DICT, (tos -- tos)) {
if (sym_matches_type(tos, &PyDict_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(tos, &PyDict_Type);
}
op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) {
if (sym_matches_type(nos, &PyDict_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(nos, &PyDict_Type);
}
@@ -1346,48 +1346,48 @@ dummy_func(void) {
if (sym_matches_type(tos, &PySet_Type) ||
sym_matches_type(tos, &PyFrozenSet_Type))
{
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
}
op(_GUARD_NOS_NULL, (null, unused -- null, unused)) {
if (sym_is_null(null)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_null(null);
}
op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) {
if (sym_is_not_null(nos)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_non_null(nos);
}
op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) {
if (sym_is_null(null)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_null(null);
}
op(_GUARD_CALLABLE_TYPE_1, (callable, unused, unused -- callable, unused,
unused)) {
if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, (PyObject *)&PyType_Type);
}
op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused,
unused)) {
if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, (PyObject *)&PyTuple_Type);
}
op(_GUARD_CALLABLE_STR_1, (callable, unused, unused -- callable, unused,
unused)) {
if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, (PyObject *)&PyUnicode_Type);
}
@@ -1401,7 +1401,7 @@ dummy_func(void) {
goto error;
}
if (_Py_IsImmortal(temp)) {
- REPLACE_OP(this_instr, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW,
+ ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW,
0, (uintptr_t)temp);
}
res = sym_new_const(ctx, temp);
@@ -1423,7 +1423,7 @@ dummy_func(void) {
goto error;
}
if (_Py_IsImmortal(temp)) {
- REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)temp);
+ ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp);
}
len = sym_new_const(ctx, temp);
Py_DECREF(temp);
@@ -1433,7 +1433,7 @@ dummy_func(void) {
op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused,
unused)) {
PyObject *len = _PyInterpreterState_GET()->callable_cache.len;
if (sym_get_const(ctx, callable) == len) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, len);
}
@@ -1441,7 +1441,7 @@ dummy_func(void) {
op(_GUARD_CALLABLE_ISINSTANCE, (callable, unused, unused, unused --
callable, unused, unused, unused)) {
PyObject *isinstance =
_PyInterpreterState_GET()->callable_cache.isinstance;
if (sym_get_const(ctx, callable) == isinstance) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, isinstance);
}
@@ -1449,7 +1449,7 @@ dummy_func(void) {
op(_GUARD_CALLABLE_LIST_APPEND, (callable, unused, unused -- callable,
unused, unused)) {
PyObject *list_append =
_PyInterpreterState_GET()->callable_cache.list_append;
if (sym_get_const(ctx, callable) == list_append) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, list_append);
}
@@ -1485,7 +1485,7 @@ dummy_func(void) {
ctx->frame->globals_watched = true;
}
if (ctx->frame->globals_checked_version == version) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
}
}
@@ -1546,7 +1546,7 @@ dummy_func(void) {
ctx->frame->globals_watched = true;
}
if (ctx->frame->globals_checked_version != version &&
this_instr[-1].opcode == _NOP) {
- REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0,
version);
+ REPLACE_OP(&ctx->out_buffer[ctx->out_len - 1],
_GUARD_GLOBALS_VERSION, 0, version);
ctx->frame->globals_checked_version = version;
}
if (ctx->frame->globals_checked_version == version) {
diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index e737a0b8800b2c..012fe16bfd9096 100644
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -76,7 +76,7 @@
JitOptRef value;
PyCodeObject *co = get_current_code_object(ctx);
PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg);
- REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)val);
+ ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);
value = PyJitRef_Borrow(sym_new_const(ctx, val));
CHECK_STACK_BOUNDS(1);
stack_pointer[0] = value;
@@ -90,7 +90,7 @@
PyObject *val = PyLong_FromLong(oparg);
assert(val);
assert(_Py_IsImmortal(val));
- REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)val);
+ ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val);
value = PyJitRef_Borrow(sym_new_const(ctx, val));
CHECK_STACK_BOUNDS(1);
stack_pointer[0] = value;
@@ -117,16 +117,16 @@
if (PyJitRef_IsBorrowed(value) ||
sym_is_immortal(PyJitRef_Unwrap(value)) ||
sym_is_null(value)) {
- REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0);
+ ADD_OP(_POP_TOP_NOP, 0, 0);
}
else if (typ == &PyLong_Type) {
- REPLACE_OP(this_instr, _POP_TOP_INT, 0, 0);
+ ADD_OP(_POP_TOP_INT, 0, 0);
}
else if (typ == &PyFloat_Type) {
- REPLACE_OP(this_instr, _POP_TOP_FLOAT, 0, 0);
+ ADD_OP(_POP_TOP_FLOAT, 0, 0);
}
else if (typ == &PyUnicode_Type) {
- REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0);
+ ADD_OP(_POP_TOP_UNICODE, 0, 0);
}
CHECK_STACK_BOUNDS(-1);
stack_pointer += -1;
@@ -145,7 +145,7 @@
JitOptRef value;
value = stack_pointer[-1];
if (PyJitRef_IsBorrowed(value)) {
- REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0);
+ ADD_OP(_POP_TOP_NOP, 0, 0);
}
CHECK_STACK_BOUNDS(-1);
stack_pointer += -1;
@@ -157,7 +157,7 @@
JitOptRef value;
value = stack_pointer[-1];
if (PyJitRef_IsBorrowed(value)) {
- REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0);
+ ADD_OP(_POP_TOP_NOP, 0, 0);
}
CHECK_STACK_BOUNDS(-1);
stack_pointer += -1;
@@ -169,7 +169,7 @@
JitOptRef value;
value = stack_pointer[-1];
if (PyJitRef_IsBorrowed(value)) {
- REPLACE_OP(this_instr, _POP_TOP_NOP, 0, 0);
+ ADD_OP(_POP_TOP_NOP, 0, 0);
}
CHECK_STACK_BOUNDS(-1);
stack_pointer += -1;
@@ -245,7 +245,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _INSERT_1_LOAD_CONST_INLINE_BORROW
since we have one input and an immortal result
- REPLACE_OP(this_instr,
_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(1);
@@ -295,7 +295,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW
since we have one input and an immortal result
- REPLACE_OP(this_instr,
_POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_POP_TOP_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
stack_pointer[-1] = res;
@@ -353,7 +353,7 @@
JitOptRef nos;
nos = stack_pointer[-2];
if (sym_matches_type(nos, &PyList_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(nos, &PyList_Type);
break;
@@ -363,7 +363,7 @@
JitOptRef tos;
tos = stack_pointer[-1];
if (sym_matches_type(tos, &PyList_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(tos, &PyList_Type);
break;
@@ -415,7 +415,7 @@
JitOptRef nos;
nos = stack_pointer[-2];
if (sym_matches_type(nos, &PyUnicode_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(nos, &PyUnicode_Type);
break;
@@ -425,7 +425,7 @@
JitOptRef value;
value = stack_pointer[-1];
if (sym_matches_type(value, &PyUnicode_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(value, &PyUnicode_Type);
break;
@@ -454,7 +454,7 @@
JitOptRef res;
JitOptRef v;
value = stack_pointer[-1];
- REPLACE_OP(this_instr, _INSERT_1_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)Py_True);
+ ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True);
res = sym_new_const(ctx, Py_True);
v = value;
CHECK_STACK_BOUNDS(1);
@@ -493,7 +493,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _INSERT_1_LOAD_CONST_INLINE_BORROW
since we have one input and an immortal result
- REPLACE_OP(this_instr,
_INSERT_1_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_INSERT_1_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(1);
@@ -522,11 +522,11 @@
JitOptRef left;
left = stack_pointer[-2];
if (sym_is_compact_int(left)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
else {
if (sym_get_type(left) == &PyLong_Type) {
- REPLACE_OP(this_instr, _GUARD_NOS_OVERFLOWED, 0, 0);
+ ADD_OP(_GUARD_NOS_OVERFLOWED, 0, 0);
}
sym_set_compact_int(left);
}
@@ -537,11 +537,11 @@
JitOptRef value;
value = stack_pointer[-1];
if (sym_is_compact_int(value)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
else {
if (sym_get_type(value) == &PyLong_Type) {
- REPLACE_OP(this_instr, _GUARD_TOS_OVERFLOWED, 0, 0);
+ ADD_OP(_GUARD_TOS_OVERFLOWED, 0, 0);
}
sym_set_compact_int(value);
}
@@ -600,7 +600,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW
since we have two inputs and an immortal result
- REPLACE_OP(this_instr,
_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(1);
@@ -664,7 +664,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW
since we have two inputs and an immortal result
- REPLACE_OP(this_instr,
_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(1);
@@ -728,7 +728,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW
since we have two inputs and an immortal result
- REPLACE_OP(this_instr,
_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(1);
@@ -752,7 +752,7 @@
JitOptRef left;
left = stack_pointer[-2];
if (sym_matches_type(left, &PyFloat_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(left, &PyFloat_Type);
break;
@@ -762,7 +762,7 @@
JitOptRef value;
value = stack_pointer[-1];
if (sym_matches_type(value, &PyFloat_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(value, &PyFloat_Type);
break;
@@ -993,7 +993,7 @@
JitOptRef nos;
nos = stack_pointer[-2];
if (sym_matches_type(nos, &PyTuple_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(nos, &PyTuple_Type);
break;
@@ -1003,7 +1003,7 @@
JitOptRef tos;
tos = stack_pointer[-1];
if (sym_matches_type(tos, &PyTuple_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(tos, &PyTuple_Type);
break;
@@ -1021,7 +1021,7 @@
assert(index >= 0);
int tuple_length = sym_tuple_length(tuple_st);
if (tuple_length != -1 && index < tuple_length) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
}
break;
@@ -1067,7 +1067,7 @@
JitOptRef nos;
nos = stack_pointer[-2];
if (sym_matches_type(nos, &PyDict_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(nos, &PyDict_Type);
break;
@@ -1077,7 +1077,7 @@
JitOptRef tos;
tos = stack_pointer[-1];
if (sym_matches_type(tos, &PyDict_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(tos, &PyDict_Type);
break;
@@ -1499,11 +1499,11 @@
JitOptRef *null;
null = &stack_pointer[0];
if (oparg & 1) {
- REPLACE_OP(this_instr, _PUSH_NULL, 0, 0);
+ ADD_OP(_PUSH_NULL, 0, 0);
null[0] = sym_new_null(ctx);
}
else {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
CHECK_STACK_BOUNDS((oparg & 1));
stack_pointer += (oparg & 1);
@@ -1528,7 +1528,7 @@
ctx->frame->globals_watched = true;
}
if (ctx->frame->globals_checked_version == version) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
}
}
@@ -1557,7 +1557,7 @@
ctx->frame->globals_watched = true;
}
if (ctx->frame->globals_checked_version != version &&
this_instr[-1].opcode == _NOP) {
- REPLACE_OP(this_instr-1, _GUARD_GLOBALS_VERSION, 0,
version);
+ REPLACE_OP(&ctx->out_buffer[ctx->out_len - 1],
_GUARD_GLOBALS_VERSION, 0, version);
ctx->frame->globals_checked_version = version;
}
if (ctx->frame->globals_checked_version == version) {
@@ -1823,7 +1823,7 @@
uint32_t type_version = (uint32_t)this_instr->operand0;
assert(type_version);
if (sym_matches_type_version(owner, type_version)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
} else {
PyTypeObject *type = _PyType_LookupByVersion(type_version);
if (type) {
@@ -1937,7 +1937,7 @@
PyObject *type = (PyObject *)_PyType_LookupByVersion(type_version);
if (type) {
if (type == sym_get_const(ctx, owner)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
else {
sym_set_const(owner, type);
@@ -2077,7 +2077,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW
since we have two inputs and an immortal result
- REPLACE_OP(this_instr,
_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(-1);
@@ -2139,7 +2139,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW
since we have two inputs and an immortal result
- REPLACE_OP(this_instr,
_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(1);
@@ -2203,7 +2203,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW
since we have two inputs and an immortal result
- REPLACE_OP(this_instr,
_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(1);
@@ -2265,7 +2265,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW
since we have two inputs and an immortal result
- REPLACE_OP(this_instr,
_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(1);
@@ -2345,7 +2345,7 @@
PyObject *result = sym_get_const(ctx, b);
if (_Py_IsImmortal(result)) {
// Replace with _INSERT_2_LOAD_CONST_INLINE_BORROW
since we have two inputs and an immortal result
- REPLACE_OP(this_instr,
_INSERT_2_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_INSERT_2_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(1);
@@ -2371,7 +2371,7 @@
if (sym_matches_type(tos, &PySet_Type) ||
sym_matches_type(tos, &PyFrozenSet_Type))
{
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
break;
}
@@ -2481,7 +2481,7 @@
goto error;
}
if (_Py_IsImmortal(temp)) {
- REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)temp);
+ ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp);
}
len = sym_new_const(ctx, temp);
CHECK_STACK_BOUNDS(1);
@@ -2606,7 +2606,7 @@
JitOptRef iter;
iter = stack_pointer[-2];
if (sym_matches_type(iter, &PyTuple_Type)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_type(iter, &PyTuple_Type);
break;
@@ -2860,8 +2860,8 @@
uint32_t func_version = (uint32_t)this_instr->operand0;
if (sym_is_const(ctx, callable) && sym_matches_type(callable,
&PyFunction_Type)) {
assert(PyFunction_Check(sym_get_const(ctx, callable)));
- REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0,
func_version);
- this_instr->operand1 = (uintptr_t)sym_get_const(ctx, callable);
+ ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version);
+ ctx->out_buffer[ctx->out_len - 1].operand1 =
(uintptr_t)sym_get_const(ctx, callable);
}
sym_set_type(callable, &PyFunction_Type);
break;
@@ -2878,8 +2878,8 @@
if (sym_is_const(ctx, callable) && sym_matches_type(callable,
&PyMethod_Type)) {
PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx,
callable);
assert(PyMethod_Check(method));
- REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0,
func_version);
- this_instr->operand1 = (uintptr_t)method->im_func;
+ ADD_OP(_CHECK_FUNCTION_VERSION_INLINE, 0, func_version);
+ ctx->out_buffer[ctx->out_len - 1].operand1 =
(uintptr_t)method->im_func;
}
sym_set_type(callable, &PyMethod_Type);
break;
@@ -2927,7 +2927,7 @@
case _CHECK_PEP_523: {
if (_PyInterpreterState_GET()->eval_frame == NULL) {
- REPLACE_OP(this_instr, _NOP, 0 ,0);
+ ADD_OP(_NOP, 0 ,0);
}
break;
}
@@ -2943,7 +2943,7 @@
PyFunctionObject *func = (PyFunctionObject
*)sym_get_const(ctx, callable);
PyCodeObject *co = (PyCodeObject *)func->func_code;
if (co->co_argcount == oparg + !sym_is_null(self_or_null))
{
- REPLACE_OP(this_instr, _NOP, 0 ,0);
+ ADD_OP(_NOP, 0 ,0);
}
}
}
@@ -3021,7 +3021,7 @@
JitOptRef null;
null = stack_pointer[-2];
if (sym_is_null(null)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_null(null);
break;
@@ -3031,7 +3031,7 @@
JitOptRef nos;
nos = stack_pointer[-2];
if (sym_is_not_null(nos)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_non_null(nos);
break;
@@ -3041,7 +3041,7 @@
JitOptRef null;
null = stack_pointer[-3];
if (sym_is_null(null)) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_null(null);
break;
@@ -3051,7 +3051,7 @@
JitOptRef callable;
callable = stack_pointer[-3];
if (sym_get_const(ctx, callable) == (PyObject *)&PyType_Type) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, (PyObject *)&PyType_Type);
break;
@@ -3065,8 +3065,8 @@
PyObject* type = (PyObject *)sym_get_type(arg);
if (type) {
res = sym_new_const(ctx, type);
- REPLACE_OP(this_instr, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0,
- (uintptr_t)type);
+ ADD_OP(_SHUFFLE_2_LOAD_CONST_INLINE_BORROW, 0,
+ (uintptr_t)type);
}
else {
res = sym_new_not_null(ctx);
@@ -3084,7 +3084,7 @@
JitOptRef callable;
callable = stack_pointer[-3];
if (sym_get_const(ctx, callable) == (PyObject *)&PyUnicode_Type) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, (PyObject *)&PyUnicode_Type);
break;
@@ -3114,7 +3114,7 @@
JitOptRef callable;
callable = stack_pointer[-3];
if (sym_get_const(ctx, callable) == (PyObject *)&PyTuple_Type) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, (PyObject *)&PyTuple_Type);
break;
@@ -3256,7 +3256,7 @@
callable = stack_pointer[-3];
PyObject *len = _PyInterpreterState_GET()->callable_cache.len;
if (sym_get_const(ctx, callable) == len) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, len);
break;
@@ -3278,8 +3278,8 @@
goto error;
}
if (_Py_IsImmortal(temp)) {
- REPLACE_OP(this_instr, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW,
- 0, (uintptr_t)temp);
+ ADD_OP(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW,
+ 0, (uintptr_t)temp);
}
res = sym_new_const(ctx, temp);
CHECK_STACK_BOUNDS(-2);
@@ -3302,7 +3302,7 @@
callable = stack_pointer[-4];
PyObject *isinstance =
_PyInterpreterState_GET()->callable_cache.isinstance;
if (sym_get_const(ctx, callable) == isinstance) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, isinstance);
break;
@@ -3323,7 +3323,7 @@
out = Py_True;
}
sym_set_const(res, out);
- REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW,
0, (uintptr_t)out);
+ ADD_OP(_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)out);
}
CHECK_STACK_BOUNDS(-3);
stack_pointer[-4] = res;
@@ -3337,7 +3337,7 @@
callable = stack_pointer[-3];
PyObject *list_append =
_PyInterpreterState_GET()->callable_cache.list_append;
if (sym_get_const(ctx, callable) == list_append) {
- REPLACE_OP(this_instr, _NOP, 0, 0);
+ ADD_OP(_NOP, 0, 0);
}
sym_set_const(callable, list_append);
break;
@@ -3631,7 +3631,7 @@
PyObject *result = sym_get_const(ctx, res);
if (_Py_IsImmortal(result)) {
// Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW
since we have two inputs and an immortal result
- REPLACE_OP(this_instr,
_POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result);
+ ADD_OP(_POP_TWO_LOAD_CONST_INLINE_BORROW, 0,
(uintptr_t)result);
}
}
CHECK_STACK_BOUNDS(-1);
@@ -3718,7 +3718,7 @@
if (sym_is_const(ctx, flag)) {
PyObject *value = sym_get_const(ctx, flag);
assert(value != NULL);
- eliminate_pop_guard(this_instr, value != Py_True);
+ eliminate_pop_guard(this_instr, ctx, value != Py_True);
}
sym_set_const(flag, Py_True);
CHECK_STACK_BOUNDS(-1);
@@ -3733,7 +3733,7 @@
if (sym_is_const(ctx, flag)) {
PyObject *value = sym_get_const(ctx, flag);
assert(value != NULL);
- eliminate_pop_guard(this_instr, value != Py_False);
+ eliminate_pop_guard(this_instr, ctx, value != Py_False);
}
sym_set_const(flag, Py_False);
CHECK_STACK_BOUNDS(-1);
@@ -3748,11 +3748,11 @@
if (sym_is_const(ctx, val)) {
PyObject *value = sym_get_const(ctx, val);
assert(value != NULL);
- eliminate_pop_guard(this_instr, !Py_IsNone(value));
+ eliminate_pop_guard(this_instr, ctx, !Py_IsNone(value));
}
else if (sym_has_type(val)) {
assert(!sym_matches_type(val, &_PyNone_Type));
- eliminate_pop_guard(this_instr, true);
+ eliminate_pop_guard(this_instr, ctx, true);
}
sym_set_const(val, Py_None);
CHECK_STACK_BOUNDS(-1);
@@ -3767,11 +3767,11 @@
if (sym_is_const(ctx, val)) {
PyObject *value = sym_get_const(ctx, val);
assert(value != NULL);
- eliminate_pop_guard(this_instr, Py_IsNone(value));
+ eliminate_pop_guard(this_instr, ctx, Py_IsNone(value));
}
else if (sym_has_type(val)) {
assert(!sym_matches_type(val, &_PyNone_Type));
- eliminate_pop_guard(this_instr, false);
+ eliminate_pop_guard(this_instr, ctx, false);
}
CHECK_STACK_BOUNDS(-1);
stack_pointer += -1;
diff --git a/Tools/cases_generator/optimizer_generator.py
b/Tools/cases_generator/optimizer_generator.py
index 392aad86549e62..65896221ba7a05 100644
--- a/Tools/cases_generator/optimizer_generator.py
+++ b/Tools/cases_generator/optimizer_generator.py
@@ -270,7 +270,7 @@ def replace_opcode_if_evaluates_pure(
emitter.emit(f"PyObject *result = sym_get_const(ctx,
{output_identifier.text});\n")
emitter.emit(f"if (_Py_IsImmortal(result)) {{\n")
emitter.emit(f"// Replace with {replacement_uop} since we have
{input_desc} and an immortal result\n")
- emitter.emit(f"REPLACE_OP(this_instr, {replacement_uop}, 0,
(uintptr_t)result);\n")
+ emitter.emit(f"ADD_OP({replacement_uop}, 0,
(uintptr_t)result);\n")
emitter.emit("}\n")
emitter.emit("}\n")
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
