From bd94999a58a6ece698cc3ef2d4318fc9640d4e1f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 30 May 2024 16:09:05 +0100 Subject: [PATCH 1/7] Experiment to measure maximum objects and space deferred without adding extra eval breaker checks --- Include/cpython/pystate.h | 3 + Include/cpython/pystats.h | 19 +++++ Include/internal/pycore_code.h | 3 + Include/internal/pycore_frame.h | 4 + Include/object.h | 8 ++ Include/pystats.h | 1 + Objects/object.c | 25 ++++++- Python/bytecodes.c | 117 ++++++++++++++++++++++++++++-- Python/ceval.c | 1 + Python/ceval_macros.h | 14 +++- Python/executor_cases.c.h | 80 +++++++++++++++++++- Python/generated_cases.c.h | 125 +++++++++++++++++++++++++++++--- Python/pystate.c | 6 ++ Python/specialize.c | 5 ++ 14 files changed, 387 insertions(+), 24 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ed3ee090ae53db..93dce912be0231 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -191,6 +191,9 @@ struct _ts { PyObject *previous_executor; uint64_t dict_global_version; + + int sp_cached; + int in_gc; }; #ifdef Py_DEBUG diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 38480a4f6cd78f..adfd45da574615 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -87,8 +87,12 @@ typedef struct _object_stats { uint64_t type_cache_dunder_hits; uint64_t type_cache_dunder_misses; uint64_t type_cache_collisions; + uint64_t max_deferred_space; + uint64_t max_deferred_objects; /* Temporary value used during GC */ uint64_t object_visits; + uint64_t deferred_space; + uint64_t deferred_objects; } ObjectStats; typedef struct _gc_stats { @@ -167,3 +171,18 @@ PyAPI_DATA(PyStats*) _Py_stats; # define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) # define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) #endif + + +#define UPDATE_DEFERRED_STATS() \ +do { \ + if (_Py_stats) { \ + if (_Py_stats->object_stats.deferred_space > _Py_stats->object_stats.max_deferred_space) { \ + _Py_stats->object_stats.max_deferred_space = _Py_stats->object_stats.deferred_space; \ + } \ + if (_Py_stats->object_stats.deferred_objects > _Py_stats->object_stats.max_deferred_objects) { \ + _Py_stats->object_stats.max_deferred_objects = _Py_stats->object_stats.deferred_objects; \ + } \ + _Py_stats->object_stats.deferred_space = 0; \ + _Py_stats->object_stats.deferred_objects = 0; \ + } \ +} while (0) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index bcbaf60f226c77..6be21000aa2387 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -307,6 +307,7 @@ extern void _Py_Specialize_ContainsOp(PyObject *value, _Py_CODEUNIT *instr); #define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) #define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) #define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) +#define OBJECT_STAT_ADD(name, value) do { if (_Py_stats) _Py_stats->object_stats.name += value; } while (0) #define OBJECT_STAT_INC_COND(name, cond) \ do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) #define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0) @@ -343,6 +344,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OPCODE_EXE_INC(opname) ((void)0) #define CALL_STAT_INC(name) ((void)0) #define OBJECT_STAT_INC(name) ((void)0) +#define OBJECT_STAT_ADD(name, val) ((void)0) #define OBJECT_STAT_INC_COND(name, cond) ((void)0) #define EVAL_CALL_STAT_INC(name) ((void)0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) @@ -354,6 +356,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OPT_ERROR_IN_OPCODE(opname) ((void)0) #define OPT_HIST(length, name) ((void)0) #define RARE_EVENT_STAT_INC(name) ((void)0) +#define UPDATE_DEFERRED_STATS() ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 994900c007f4bd..4e5d34c73206a2 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -163,6 +163,8 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { + assert(frame->stacktop >= 0); + PyThreadState_GET()->sp_cached++; PyObject **sp = frame->localsplus + frame->stacktop; frame->stacktop = -1; return sp; @@ -171,6 +173,8 @@ _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) { + assert(frame->stacktop == -1); + PyThreadState_GET()->sp_cached--; frame->stacktop = (int)(stack_pointer - frame->localsplus); } diff --git a/Include/object.h b/Include/object.h index 9132784628a501..66dbff8732f1ea 100644 --- a/Include/object.h +++ b/Include/object.h @@ -789,6 +789,8 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *); PyAPI_FUNC(void) _Py_IncRef(PyObject *); PyAPI_FUNC(void) _Py_DecRef(PyObject *); +PyAPI_FUNC(int) PyThreadState_GetSpCached(void); + static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { #if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) @@ -923,6 +925,9 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) if (op->ob_refcnt <= 0) { _Py_NegativeRefcount(filename, lineno, op); } + if (PyThreadState_GetSpCached() == 0) { + UPDATE_DEFERRED_STATS(); + } if (_Py_IsImmortal(op)) { return; } @@ -939,6 +944,9 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) { // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. + if (PyThreadState_GetSpCached() == 0) { + UPDATE_DEFERRED_STATS(); + } if (_Py_IsImmortal(op)) { return; } diff --git a/Include/pystats.h b/Include/pystats.h index acfa32201711e0..98dc5022e4620d 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -18,6 +18,7 @@ extern "C" { #else # define _Py_INCREF_STAT_INC() ((void)0) # define _Py_DECREF_STAT_INC() ((void)0) +# define UPDATE_DEFERRED_STATS() ((void)0) #endif // !Py_STATS #ifdef __cplusplus diff --git a/Objects/object.c b/Objects/object.c index d4fe14c5b3d1aa..5a1a992fa13c95 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2823,14 +2823,30 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, Py_FatalError("_PyObject_AssertFailed"); } +size_t size_guess(PyObject *op) +{ + size_t res = Py_TYPE(op)->tp_basicsize; + size_t isize = Py_TYPE(op)->tp_itemsize; + + if (isize > 0) { + size_t count = _PyVarObject_CAST(op)->ob_size; + if (Py_TYPE(op) == &PyLong_Type) { + count >>= 3; + } + if (count < 100000000) { + res += count * isize; + } + } + return res; +} void _Py_Dealloc(PyObject *op) { PyTypeObject *type = Py_TYPE(op); destructor dealloc = type->tp_dealloc; -#ifdef Py_DEBUG PyThreadState *tstate = _PyThreadState_GET(); +#ifdef Py_DEBUG PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL; // Keep the old exception type alive to prevent undefined behavior // on (tstate->curexc_type != old_exc_type) below @@ -2848,7 +2864,14 @@ _Py_Dealloc(PyObject *op) #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif + + if (tstate->sp_cached && tstate->in_gc == 0) { + OBJECT_STAT_ADD(deferred_space, size_guess(op)); + OBJECT_STAT_INC(deferred_objects); + } + tstate->in_gc++; (*dealloc)(op); + tstate->in_gc--; #ifdef Py_DEBUG // gh-89373: The tp_dealloc function must leave the current exception diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9a8198515dea5e..39ee6d595dfe5e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -184,6 +184,7 @@ dummy_func( uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); + UPDATE_DEFERRED_STATS(); DEOPT_IF(eval_breaker != version); } @@ -292,7 +293,10 @@ dummy_func( /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + SAVE_SP(); + int err = monitor_stop_iteration(tstate, frame, this_instr, value); + LOAD_SP(); + if (err) { ERROR_NO_POP(); } } @@ -305,7 +309,10 @@ dummy_func( tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + SAVE_SP(); + int err = monitor_stop_iteration(tstate, frame, this_instr, value); + LOAD_SP(); + if (err) { ERROR_NO_POP(); } } @@ -313,7 +320,9 @@ dummy_func( } inst(UNARY_NEGATIVE, (value -- res)) { + SAVE_SP(); res = PyNumber_Negative(value); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -345,7 +354,9 @@ dummy_func( } op(_TO_BOOL, (value -- res)) { + SAVE_SP(); int err = PyObject_IsTrue(value); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err < 0, error); res = err ? Py_True : Py_False; @@ -586,7 +597,9 @@ dummy_func( } op(_BINARY_SUBSCR, (container, sub -- res)) { + SAVE_SP(); res = PyObject_GetItem(container, sub); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -594,7 +607,9 @@ dummy_func( macro(BINARY_SUBSCR) = _SPECIALIZE_BINARY_SUBSCR + _BINARY_SUBSCR; inst(BINARY_SLICE, (container, start, stop -- res)) { + SAVE_SP(); PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + LOAD_SP(); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { @@ -615,7 +630,9 @@ dummy_func( err = 1; } else { + SAVE_SP(); err = PyObject_SetItem(container, slice, v); + LOAD_SP(); Py_DECREF(slice); } Py_DECREF(v); @@ -734,7 +751,9 @@ dummy_func( op(_STORE_SUBSCR, (v, container, sub -- )) { /* container[sub] = v */ + SAVE_SP(); int err = PyObject_SetItem(container, sub, v); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -770,21 +789,27 @@ dummy_func( inst(DELETE_SUBSCR, (container, sub --)) { /* del container[sub] */ + SAVE_SP(); int err = PyObject_DelItem(container, sub); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } inst(CALL_INTRINSIC_1, (value -- res)) { assert(oparg <= MAX_INTRINSIC_1); + SAVE_SP(); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } inst(CALL_INTRINSIC_2, (value2, value1 -- res)) { assert(oparg <= MAX_INTRINSIC_2); + SAVE_SP(); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -817,6 +842,7 @@ dummy_func( assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ + SAVE_SP(); tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; @@ -1028,12 +1054,14 @@ dummy_func( frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } + SAVE_SP(); if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); } else { retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } + LOAD_SP(); if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { @@ -1217,10 +1245,12 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(true, error); } + SAVE_SP(); if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1334,7 +1364,9 @@ dummy_func( op(_STORE_ATTR, (v, owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_SetAttr(owner, name, v); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1720,7 +1752,10 @@ dummy_func( } inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) { - if (PyDict_Update(dict, update) < 0) { + SAVE_SP(); + int err = PyDict_Update(dict, update); + LOAD_SP(); + if (err < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", @@ -1733,7 +1768,10 @@ dummy_func( } inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) { - if (_PyDict_MergeEx(dict, update, 2) < 0) { + SAVE_SP(); + int err = _PyDict_MergeEx(dict, update, 2); + LOAD_SP(); + if (err < 0) { _PyEval_FormatKwargsError(tstate, callable, update); DECREF_INPUTS(); ERROR_IF(true, error); @@ -1880,7 +1918,10 @@ dummy_func( if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + SAVE_SP(); + int err = _PyObject_GetMethod(owner, name, &attr); + LOAD_SP(); + if (err) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -1902,7 +1943,9 @@ dummy_func( } else { /* Classic, pushes one value. */ + SAVE_SP(); attr = PyObject_GetAttr(owner, name); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(attr == NULL, error); } @@ -2194,7 +2237,9 @@ dummy_func( op(_COMPARE_OP, (left, right -- res)) { assert((oparg >> 5) <= Py_GE); + SAVE_SP(); res = PyObject_RichCompare(left, right, oparg >> 5); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); if (oparg & 16) { @@ -2271,7 +2316,9 @@ dummy_func( }; op(_CONTAINS_OP, (left, right -- b)) { + SAVE_SP(); int res = PySequence_Contains(right, left); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res < 0, error); b = (res ^ oparg) ? Py_True : Py_False; @@ -2345,14 +2392,18 @@ dummy_func( tier1 inst(IMPORT_NAME, (level, fromlist -- res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); res = import_name(tstate, frame, name, fromlist, level); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } tier1 inst(IMPORT_FROM, (from -- from, res)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); res = import_from(tstate, from, name); + LOAD_SP(); ERROR_IF(res == NULL, error); } @@ -2576,7 +2627,9 @@ dummy_func( replaced op(_FOR_ITER, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + SAVE_SP(); next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { @@ -2811,7 +2864,9 @@ dummy_func( } ERROR_NO_POP(); } + SAVE_SP(); exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + LOAD_SP(); if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -2824,7 +2879,9 @@ dummy_func( ERROR_NO_POP(); } DECREF_INPUTS(); + SAVE_SP(); res = PyObject_CallNoArgs(enter); + LOAD_SP(); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2836,7 +2893,9 @@ dummy_func( /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ + SAVE_SP(); PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + LOAD_SP(); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -2846,7 +2905,9 @@ dummy_func( } ERROR_NO_POP(); } + SAVE_SP(); exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + LOAD_SP(); if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -2859,7 +2920,9 @@ dummy_func( ERROR_NO_POP(); } DECREF_INPUTS(); + SAVE_SP(); res = PyObject_CallNoArgs(enter); + LOAD_SP(); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -2890,8 +2953,10 @@ dummy_func( assert(PyLong_Check(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; + SAVE_SP(); res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); ERROR_IF(res == NULL, error); } @@ -3119,10 +3184,12 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; @@ -3234,10 +3301,12 @@ dummy_func( total_args++; } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(callable); for (int i = 0; i < total_args; i++) { @@ -3302,7 +3371,7 @@ dummy_func( // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); SYNC_SP(); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -3339,7 +3408,9 @@ dummy_func( DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyType_Type); STAT_INC(CALL, hit); + SAVE_SP(); res = Py_NewRef(Py_TYPE(arg)); + LOAD_SP(); Py_DECREF(arg); } @@ -3348,7 +3419,9 @@ dummy_func( DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); + SAVE_SP(); res = PyObject_Str(arg); + LOAD_SP(); Py_DECREF(arg); ERROR_IF(res == NULL, error); } @@ -3364,7 +3437,9 @@ dummy_func( DEOPT_IF(null != NULL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); + SAVE_SP(); res = PySequence_Tuple(arg); + LOAD_SP(); Py_DECREF(arg); ERROR_IF(res == NULL, error); } @@ -3446,7 +3521,9 @@ dummy_func( PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); + SAVE_SP(); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + LOAD_SP(); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); @@ -3477,7 +3554,9 @@ dummy_func( PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyObject *arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3504,10 +3583,12 @@ dummy_func( STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ + SAVE_SP(); res = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), args, total_args); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ @@ -3538,7 +3619,9 @@ dummy_func( PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); + SAVE_SP(); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ @@ -3567,7 +3650,9 @@ dummy_func( DEOPT_IF(callable != interp->callable_cache.len); STAT_INC(CALL, hit); PyObject *arg = args[0]; + SAVE_SP(); Py_ssize_t len_i = PyObject_Length(arg); + LOAD_SP(); if (len_i < 0) { ERROR_NO_POP(); } @@ -3593,7 +3678,9 @@ dummy_func( STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; + SAVE_SP(); int retval = PyObject_IsInstance(inst, cls); + LOAD_SP(); if (retval < 0) { ERROR_NO_POP(); } @@ -3646,7 +3733,9 @@ dummy_func( STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -3678,7 +3767,9 @@ dummy_func( int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + SAVE_SP(); res = cfunc(self, args + 1, nargs, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ @@ -3714,7 +3805,9 @@ dummy_func( STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -3745,7 +3838,9 @@ dummy_func( PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; + SAVE_SP(); res = cfunc(self, args + 1, nargs); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { @@ -3816,10 +3911,12 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); + LOAD_SP(); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; @@ -3913,7 +4010,9 @@ dummy_func( frame->return_offset = 1; DISPATCH_INLINED(new_frame); } + SAVE_SP(); result = PyObject_Call(func, callargs, kwargs); + LOAD_SP(); } DECREF_INPUTS(); assert(PEEK(2 + (oparg & 1)) == NULL); @@ -3998,7 +4097,9 @@ dummy_func( conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; + SAVE_SP(); result = conv_fn(value); + LOAD_SP(); Py_DECREF(value); ERROR_IF(result == NULL, error); } @@ -4044,7 +4145,9 @@ dummy_func( op(_BINARY_OP, (lhs, rhs -- res)) { assert(_PyEval_BinaryOps[oparg]); + SAVE_SP(); res = _PyEval_BinaryOps[oparg](lhs, rhs); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -4057,8 +4160,10 @@ dummy_func( } inst(INSTRUMENTED_INSTRUCTION, ( -- )) { + SAVE_SP(); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); + LOAD_SP(); ERROR_IF(next_opcode < 0, error); next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { diff --git a/Python/ceval.c b/Python/ceval.c index 324d062fe9bb43..399503f7172767 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -765,6 +765,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int next_instr = frame->instr_ptr; resume_frame: + // assert(tstate->sp_cached == 0 || tstate->in_gc); stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef LLTRACE diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 50941e4ec473e8..e896279c099f4f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -137,10 +137,14 @@ do { \ _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); \ QSBR_QUIESCENT_STATE(tstate); \ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { \ - if (_Py_HandlePending(tstate) != 0) { \ + SAVE_SP(); \ + int err = _Py_HandlePending(tstate); \ + LOAD_SP(); \ + if (err != 0) { \ GOTO_ERROR(error); \ } \ - } + } \ + UPDATE_DEFERRED_STATS(); /* Tuple access macros */ @@ -400,7 +404,10 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { /* There's no STORE_IP(), it's inlined by the code generator. */ #define LOAD_SP() \ -stack_pointer = _PyFrame_GetStackPointer(frame); +stack_pointer = _PyFrame_GetStackPointer(frame) + +#define SAVE_SP() \ +_PyFrame_SetStackPointer(frame, stack_pointer) /* Tier-switching macros. */ @@ -447,3 +454,4 @@ do { \ #define EXIT_TO_TRACE() goto exit_to_trace #define EXIT_TO_TIER1() goto exit_to_tier1 #define EXIT_TO_TIER1_DYNAMIC() goto exit_to_tier1_dynamic; + diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e862364cb23e7a..6e5f1cca9471f8 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -23,6 +23,7 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); + UPDATE_DEFERRED_STATS(); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -297,7 +298,9 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + SAVE_SP(); res = PyNumber_Negative(value); + LOAD_SP(); Py_DECREF(value); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; @@ -318,7 +321,9 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + SAVE_SP(); int err = PyObject_IsTrue(value); + LOAD_SP(); Py_DECREF(value); if (err < 0) JUMP_TO_ERROR(); res = err ? Py_True : Py_False; @@ -637,7 +642,9 @@ PyObject *res; sub = stack_pointer[-1]; container = stack_pointer[-2]; + SAVE_SP(); res = PyObject_GetItem(container, sub); + LOAD_SP(); Py_DECREF(container); Py_DECREF(sub); if (res == NULL) JUMP_TO_ERROR(); @@ -654,7 +661,9 @@ stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; + SAVE_SP(); PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + LOAD_SP(); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { @@ -686,7 +695,9 @@ err = 1; } else { + SAVE_SP(); err = PyObject_SetItem(container, slice, v); + LOAD_SP(); Py_DECREF(slice); } Py_DECREF(v); @@ -862,7 +873,9 @@ container = stack_pointer[-2]; v = stack_pointer[-3]; /* container[sub] = v */ + SAVE_SP(); int err = PyObject_SetItem(container, sub, v); + LOAD_SP(); Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); @@ -933,7 +946,9 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ + SAVE_SP(); int err = PyObject_DelItem(container, sub); + LOAD_SP(); Py_DECREF(container); Py_DECREF(sub); if (err) JUMP_TO_ERROR(); @@ -947,7 +962,9 @@ oparg = CURRENT_OPARG(); value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); + SAVE_SP(); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + LOAD_SP(); Py_DECREF(value); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; @@ -962,7 +979,9 @@ value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); + SAVE_SP(); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + LOAD_SP(); Py_DECREF(value2); Py_DECREF(value1); if (res == NULL) JUMP_TO_ERROR(); @@ -1207,10 +1226,12 @@ Py_DECREF(v); if (true) JUMP_TO_ERROR(); } + SAVE_SP(); if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); + LOAD_SP(); Py_DECREF(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; @@ -1343,7 +1364,9 @@ owner = stack_pointer[-1]; v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_SetAttr(owner, name, v); + LOAD_SP(); Py_DECREF(v); Py_DECREF(owner); if (err) JUMP_TO_ERROR(); @@ -1806,7 +1829,10 @@ oparg = CURRENT_OPARG(); update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { + SAVE_SP(); + int err = PyDict_Update(dict, update); + LOAD_SP(); + if (err < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", @@ -1828,7 +1854,10 @@ update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { + SAVE_SP(); + int err = _PyDict_MergeEx(dict, update, 2); + LOAD_SP(); + if (err < 0) { _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); if (true) JUMP_TO_ERROR(); @@ -1939,7 +1968,10 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + SAVE_SP(); + int err = _PyObject_GetMethod(owner, name, &attr); + LOAD_SP(); + if (err) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -1961,7 +1993,9 @@ } else { /* Classic, pushes one value. */ + SAVE_SP(); attr = PyObject_GetAttr(owner, name); + LOAD_SP(); Py_DECREF(owner); if (attr == NULL) JUMP_TO_ERROR(); } @@ -2365,7 +2399,9 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; assert((oparg >> 5) <= Py_GE); + SAVE_SP(); res = PyObject_RichCompare(left, right, oparg >> 5); + LOAD_SP(); Py_DECREF(left); Py_DECREF(right); if (res == NULL) JUMP_TO_ERROR(); @@ -2477,7 +2513,9 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; + SAVE_SP(); int res = PySequence_Contains(right, left); + LOAD_SP(); Py_DECREF(left); Py_DECREF(right); if (res < 0) JUMP_TO_ERROR(); @@ -2941,8 +2979,10 @@ assert(PyLong_Check(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; + SAVE_SP(); res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[0] = res; stack_pointer += 1; @@ -3237,10 +3277,12 @@ total_args++; } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(callable); for (int i = 0; i < total_args; i++) { @@ -3477,7 +3519,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); stack_pointer += -1; - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -3507,7 +3549,9 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); + SAVE_SP(); res = Py_NewRef(Py_TYPE(arg)); + LOAD_SP(); Py_DECREF(arg); stack_pointer[-3] = res; stack_pointer += -2; @@ -3533,7 +3577,9 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); + SAVE_SP(); res = PyObject_Str(arg); + LOAD_SP(); Py_DECREF(arg); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; @@ -3560,7 +3606,9 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); + SAVE_SP(); res = PySequence_Tuple(arg); + LOAD_SP(); Py_DECREF(arg); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; @@ -3608,7 +3656,9 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); + SAVE_SP(); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + LOAD_SP(); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); @@ -3656,7 +3706,9 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyObject *arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(arg); @@ -3693,10 +3745,12 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ + SAVE_SP(); res = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), args, total_args); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { @@ -3737,7 +3791,9 @@ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); + SAVE_SP(); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { @@ -3776,7 +3832,9 @@ } STAT_INC(CALL, hit); PyObject *arg = args[0]; + SAVE_SP(); Py_ssize_t len_i = PyObject_Length(arg); + LOAD_SP(); if (len_i < 0) { JUMP_TO_ERROR(); } @@ -3819,7 +3877,9 @@ STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; + SAVE_SP(); int retval = PyObject_IsInstance(inst, cls); + LOAD_SP(); if (retval < 0) { JUMP_TO_ERROR(); } @@ -3878,7 +3938,9 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -3924,7 +3986,9 @@ int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + SAVE_SP(); res = cfunc(self, args + 1, nargs, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { @@ -3979,7 +4043,9 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -4024,7 +4090,9 @@ PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; + SAVE_SP(); res = cfunc(self, args + 1, nargs); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { @@ -4153,7 +4221,9 @@ conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; + SAVE_SP(); result = conv_fn(value); + LOAD_SP(); Py_DECREF(value); if (result == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = result; @@ -4213,7 +4283,9 @@ rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; assert(_PyEval_BinaryOps[oparg]); + SAVE_SP(); res = _PyEval_BinaryOps[oparg](lhs, rhs); + LOAD_SP(); Py_DECREF(lhs); Py_DECREF(rhs); if (res == NULL) JUMP_TO_ERROR(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4402787d96f12e..cc2cec10dc517e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -27,7 +27,9 @@ } goto error; } + SAVE_SP(); exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__aexit__)); + LOAD_SP(); if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -40,7 +42,9 @@ goto error; } Py_DECREF(mgr); + SAVE_SP(); res = PyObject_CallNoArgs(enter); + LOAD_SP(); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -63,7 +67,9 @@ /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ + SAVE_SP(); PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__enter__)); + LOAD_SP(); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -73,7 +79,9 @@ } goto error; } + SAVE_SP(); exit = _PyObject_LookupSpecial(mgr, &_Py_ID(__exit__)); + LOAD_SP(); if (exit == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -86,7 +94,9 @@ goto error; } Py_DECREF(mgr); + SAVE_SP(); res = PyObject_CallNoArgs(enter); + LOAD_SP(); Py_DECREF(enter); if (res == NULL) { Py_DECREF(exit); @@ -129,7 +139,9 @@ // _BINARY_OP { assert(_PyEval_BinaryOps[oparg]); + SAVE_SP(); res = _PyEval_BinaryOps[oparg](lhs, rhs); + LOAD_SP(); Py_DECREF(lhs); Py_DECREF(rhs); if (res == NULL) goto pop_2_error; @@ -398,7 +410,9 @@ stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; + SAVE_SP(); PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + LOAD_SP(); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. if (slice == NULL) { @@ -443,7 +457,9 @@ } // _BINARY_SUBSCR { + SAVE_SP(); res = PyObject_GetItem(container, sub); + LOAD_SP(); Py_DECREF(container); Py_DECREF(sub); if (res == NULL) goto pop_2_error; @@ -810,10 +826,12 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; @@ -995,7 +1013,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -1086,7 +1104,7 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -1123,7 +1141,9 @@ PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); + SAVE_SP(); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + LOAD_SP(); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); @@ -1167,10 +1187,12 @@ STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ + SAVE_SP(); res = ((PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), args, total_args); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { @@ -1217,7 +1239,9 @@ PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); + SAVE_SP(); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { @@ -1266,7 +1290,9 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyObject *arg = args[0]; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(arg); @@ -1355,7 +1381,9 @@ frame->return_offset = 1; DISPATCH_INLINED(new_frame); } + SAVE_SP(); result = PyObject_Call(func, callargs, kwargs); + LOAD_SP(); } Py_DECREF(func); Py_DECREF(callargs); @@ -1376,7 +1404,9 @@ PyObject *res; value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); + SAVE_SP(); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); + LOAD_SP(); Py_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; @@ -1393,7 +1423,9 @@ value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); + SAVE_SP(); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + LOAD_SP(); Py_DECREF(value2); Py_DECREF(value1); if (res == NULL) goto pop_2_error; @@ -1428,7 +1460,9 @@ STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; + SAVE_SP(); int retval = PyObject_IsInstance(inst, cls); + LOAD_SP(); if (retval < 0) { goto error; } @@ -1502,10 +1536,12 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); + LOAD_SP(); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; @@ -1561,7 +1597,9 @@ DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); PyObject *arg = args[0]; + SAVE_SP(); Py_ssize_t len_i = PyObject_Length(arg); + LOAD_SP(); if (len_i < 0) { goto error; } @@ -1640,7 +1678,9 @@ PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; + SAVE_SP(); res = cfunc(self, args + 1, nargs); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { @@ -1690,7 +1730,9 @@ int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + SAVE_SP(); res = cfunc(self, args + 1, nargs, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { @@ -1742,7 +1784,9 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -1792,7 +1836,9 @@ STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); + SAVE_SP(); res = _PyCFunction_TrampolineCall(cfunc, self, arg); + LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); @@ -1839,10 +1885,12 @@ total_args++; } /* Callable is not a normal Python function */ + SAVE_SP(); res = PyObject_Vectorcall( callable, args, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(callable); for (int i = 0; i < total_args; i++) { @@ -1925,7 +1973,7 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); stack_pointer += -2 - oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -1997,7 +2045,7 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -2029,7 +2077,9 @@ DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); + SAVE_SP(); res = PyObject_Str(arg); + LOAD_SP(); Py_DECREF(arg); if (res == NULL) goto pop_3_error; } @@ -2062,7 +2112,9 @@ DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); + SAVE_SP(); res = PySequence_Tuple(arg); + LOAD_SP(); Py_DECREF(arg); if (res == NULL) goto pop_3_error; } @@ -2093,7 +2145,9 @@ DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); + SAVE_SP(); res = Py_NewRef(Py_TYPE(arg)); + LOAD_SP(); Py_DECREF(arg); stack_pointer[-3] = res; stack_pointer += -2; @@ -2215,7 +2269,9 @@ // _COMPARE_OP { assert((oparg >> 5) <= Py_GE); + SAVE_SP(); res = PyObject_RichCompare(left, right, oparg >> 5); + LOAD_SP(); Py_DECREF(left); Py_DECREF(right); if (res == NULL) goto pop_2_error; @@ -2363,7 +2419,9 @@ } // _CONTAINS_OP { + SAVE_SP(); int res = PySequence_Contains(right, left); + LOAD_SP(); Py_DECREF(left); Py_DECREF(right); if (res < 0) goto pop_2_error; @@ -2431,7 +2489,9 @@ conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; + SAVE_SP(); result = conv_fn(value); + LOAD_SP(); Py_DECREF(value); if (result == NULL) goto pop_1_error; stack_pointer[-1] = result; @@ -2565,7 +2625,9 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; /* del container[sub] */ + SAVE_SP(); int err = PyObject_DelItem(container, sub); + LOAD_SP(); Py_DECREF(container); Py_DECREF(sub); if (err) goto pop_2_error; @@ -2583,7 +2645,10 @@ update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; - if (_PyDict_MergeEx(dict, update, 2) < 0) { + SAVE_SP(); + int err = _PyDict_MergeEx(dict, update, 2); + LOAD_SP(); + if (err < 0) { _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); if (true) goto pop_1_error; @@ -2601,7 +2666,10 @@ PyObject *dict; update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { + SAVE_SP(); + int err = PyDict_Update(dict, update); + LOAD_SP(); + if (err < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", @@ -2791,7 +2859,9 @@ // _FOR_ITER { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + SAVE_SP(); next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { @@ -2850,7 +2920,7 @@ // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_SP(); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; @@ -3189,7 +3259,9 @@ PyObject *res; from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); res = import_from(tstate, from, name); + LOAD_SP(); if (res == NULL) goto error; stack_pointer[0] = res; stack_pointer += 1; @@ -3206,7 +3278,9 @@ fromlist = stack_pointer[-1]; level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); res = import_name(tstate, frame, name, fromlist, level); + LOAD_SP(); Py_DECREF(level); Py_DECREF(fromlist); if (res == NULL) goto pop_2_error; @@ -3270,7 +3344,10 @@ /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + SAVE_SP(); + int err = monitor_stop_iteration(tstate, frame, this_instr, value); + LOAD_SP(); + if (err) { goto error; } } @@ -3289,7 +3366,10 @@ value = stack_pointer[-1]; receiver = stack_pointer[-2]; if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { - if (monitor_stop_iteration(tstate, frame, this_instr, value)) { + SAVE_SP(); + int err = monitor_stop_iteration(tstate, frame, this_instr, value); + LOAD_SP(); + if (err) { goto error; } } @@ -3337,8 +3417,10 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + SAVE_SP(); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); + LOAD_SP(); if (next_opcode < 0) goto error; next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { @@ -3581,6 +3663,7 @@ assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous frame and return. */ + SAVE_SP(); tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; @@ -3738,7 +3821,10 @@ if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { + SAVE_SP(); + int err = _PyObject_GetMethod(owner, name, &attr); + LOAD_SP(); + if (err) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN @@ -3760,7 +3846,9 @@ } else { /* Classic, pushes one value. */ + SAVE_SP(); attr = PyObject_GetAttr(owner, name); + LOAD_SP(); Py_DECREF(owner); if (attr == NULL) goto pop_1_error; } @@ -5175,6 +5263,7 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); + UPDATE_DEFERRED_STATS(); DEOPT_IF(eval_breaker != version, RESUME); DISPATCH(); } @@ -5311,12 +5400,14 @@ frame->return_offset = (uint16_t)(next_instr - this_instr + oparg); DISPATCH_INLINED(gen_frame); } + SAVE_SP(); if (Py_IsNone(v) && PyIter_Check(receiver)) { retval = Py_TYPE(receiver)->tp_iternext(receiver); } else { retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } + LOAD_SP(); if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { @@ -5486,7 +5577,9 @@ v = stack_pointer[-2]; { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_SetAttr(owner, name, v); + LOAD_SP(); Py_DECREF(v); Py_DECREF(owner); if (err) goto pop_2_error; @@ -5712,10 +5805,12 @@ Py_DECREF(v); if (true) goto pop_1_error; } + SAVE_SP(); if (PyDict_CheckExact(ns)) err = PyDict_SetItem(ns, name, v); else err = PyObject_SetItem(ns, name, v); + LOAD_SP(); Py_DECREF(v); if (err) goto pop_1_error; stack_pointer += -1; @@ -5740,7 +5835,9 @@ err = 1; } else { + SAVE_SP(); err = PyObject_SetItem(container, slice, v); + LOAD_SP(); Py_DECREF(slice); } Py_DECREF(v); @@ -5780,7 +5877,9 @@ v = stack_pointer[-3]; { /* container[sub] = v */ + SAVE_SP(); int err = PyObject_SetItem(container, sub, v); + LOAD_SP(); Py_DECREF(v); Py_DECREF(container); Py_DECREF(sub); @@ -5882,7 +5981,9 @@ /* Skip 2 cache entries */ // _TO_BOOL { + SAVE_SP(); int err = PyObject_IsTrue(value); + LOAD_SP(); Py_DECREF(value); if (err < 0) goto pop_1_error; res = err ? Py_True : Py_False; @@ -6038,7 +6139,9 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + SAVE_SP(); res = PyNumber_Negative(value); + LOAD_SP(); Py_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; @@ -6208,8 +6311,10 @@ assert(PyLong_Check(lasti)); (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; + SAVE_SP(); res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + LOAD_SP(); if (res == NULL) goto error; stack_pointer[0] = res; stack_pointer += 1; diff --git a/Python/pystate.c b/Python/pystate.c index 1ea1ad982a0ec9..34f727528ac9cd 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1508,6 +1508,8 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->state = _Py_THREAD_SUSPENDED; } + tstate->sp_cached = 0; + tstate->in_gc = 0; tstate->_status.initialized = 1; } @@ -3089,3 +3091,7 @@ _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate) } #endif } + +int PyThreadState_GetSpCached(void) { + return _PyThreadState_GET()->sp_cached; +} diff --git a/Python/specialize.c b/Python/specialize.c index 9ac428c3593f56..20789a847bece7 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -203,6 +203,8 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); + fprintf(out, "Object max deferred objects: %" PRIu64 "\n", stats->max_deferred_objects); + fprintf(out, "Object max deferred space: %" PRIu64 "\n", stats->max_deferred_space); } static void @@ -313,12 +315,15 @@ print_stats(FILE *out, PyStats *stats) print_optimization_stats(out, &stats->optimization_stats); #endif print_rare_event_stats(out, &stats->rare_event_stats); + fprintf(out, "sp cached: %d", PyThreadState_GET()->sp_cached); } void _Py_StatsOn(void) { _Py_stats = &_Py_stats_struct; + _Py_stats->object_stats.deferred_objects = 0; + _Py_stats->object_stats.deferred_space = 0; } void From e12729e35ef84effe77d88a71fb2e2eb258bedc2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 30 May 2024 16:44:54 +0100 Subject: [PATCH 2/7] Strip out instrumentation --- Include/cpython/pystate.h | 2 -- Include/internal/pycore_frame.h | 2 -- Include/object.h | 8 -------- Objects/object.c | 26 +------------------------- Python/ceval.c | 1 - Python/pystate.c | 6 ------ Python/specialize.c | 1 - 7 files changed, 1 insertion(+), 45 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 93dce912be0231..387e2e7a9a138f 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -192,8 +192,6 @@ struct _ts { uint64_t dict_global_version; - int sp_cached; - int in_gc; }; #ifdef Py_DEBUG diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 4e5d34c73206a2..a1f3a598170d18 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -164,7 +164,6 @@ static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { assert(frame->stacktop >= 0); - PyThreadState_GET()->sp_cached++; PyObject **sp = frame->localsplus + frame->stacktop; frame->stacktop = -1; return sp; @@ -174,7 +173,6 @@ static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) { assert(frame->stacktop == -1); - PyThreadState_GET()->sp_cached--; frame->stacktop = (int)(stack_pointer - frame->localsplus); } diff --git a/Include/object.h b/Include/object.h index 66dbff8732f1ea..9132784628a501 100644 --- a/Include/object.h +++ b/Include/object.h @@ -789,8 +789,6 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *); PyAPI_FUNC(void) _Py_IncRef(PyObject *); PyAPI_FUNC(void) _Py_DecRef(PyObject *); -PyAPI_FUNC(int) PyThreadState_GetSpCached(void); - static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { #if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) @@ -925,9 +923,6 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) if (op->ob_refcnt <= 0) { _Py_NegativeRefcount(filename, lineno, op); } - if (PyThreadState_GetSpCached() == 0) { - UPDATE_DEFERRED_STATS(); - } if (_Py_IsImmortal(op)) { return; } @@ -944,9 +939,6 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) { // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. - if (PyThreadState_GetSpCached() == 0) { - UPDATE_DEFERRED_STATS(); - } if (_Py_IsImmortal(op)) { return; } diff --git a/Objects/object.c b/Objects/object.c index 5a1a992fa13c95..6fbc608958fff2 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2822,31 +2822,13 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, Py_FatalError("_PyObject_AssertFailed"); } - -size_t size_guess(PyObject *op) -{ - size_t res = Py_TYPE(op)->tp_basicsize; - size_t isize = Py_TYPE(op)->tp_itemsize; - - if (isize > 0) { - size_t count = _PyVarObject_CAST(op)->ob_size; - if (Py_TYPE(op) == &PyLong_Type) { - count >>= 3; - } - if (count < 100000000) { - res += count * isize; - } - } - return res; -} - void _Py_Dealloc(PyObject *op) { PyTypeObject *type = Py_TYPE(op); destructor dealloc = type->tp_dealloc; - PyThreadState *tstate = _PyThreadState_GET(); #ifdef Py_DEBUG + PyThreadState *tstate = _PyThreadState_GET(); PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL; // Keep the old exception type alive to prevent undefined behavior // on (tstate->curexc_type != old_exc_type) below @@ -2865,13 +2847,7 @@ _Py_Dealloc(PyObject *op) _Py_ForgetReference(op); #endif - if (tstate->sp_cached && tstate->in_gc == 0) { - OBJECT_STAT_ADD(deferred_space, size_guess(op)); - OBJECT_STAT_INC(deferred_objects); - } - tstate->in_gc++; (*dealloc)(op); - tstate->in_gc--; #ifdef Py_DEBUG // gh-89373: The tp_dealloc function must leave the current exception diff --git a/Python/ceval.c b/Python/ceval.c index 399503f7172767..324d062fe9bb43 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -765,7 +765,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int next_instr = frame->instr_ptr; resume_frame: - // assert(tstate->sp_cached == 0 || tstate->in_gc); stack_pointer = _PyFrame_GetStackPointer(frame); #ifdef LLTRACE diff --git a/Python/pystate.c b/Python/pystate.c index 34f727528ac9cd..1ea1ad982a0ec9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1508,8 +1508,6 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->state = _Py_THREAD_SUSPENDED; } - tstate->sp_cached = 0; - tstate->in_gc = 0; tstate->_status.initialized = 1; } @@ -3091,7 +3089,3 @@ _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate) } #endif } - -int PyThreadState_GetSpCached(void) { - return _PyThreadState_GET()->sp_cached; -} diff --git a/Python/specialize.c b/Python/specialize.c index 20789a847bece7..23b91f8713e646 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -315,7 +315,6 @@ print_stats(FILE *out, PyStats *stats) print_optimization_stats(out, &stats->optimization_stats); #endif print_rare_event_stats(out, &stats->rare_event_stats); - fprintf(out, "sp cached: %d", PyThreadState_GET()->sp_cached); } void From f748355a90b5345f028df1c9976ebc54674c0fff Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 May 2024 10:24:08 +0100 Subject: [PATCH 3/7] Convert stacktop to pointer --- Include/internal/pycore_frame.h | 47 ++++++++++++++++++--------------- Objects/frameobject.c | 18 ++++++++----- Python/ceval.c | 2 +- Python/frame.c | 22 ++++++++------- 4 files changed, 50 insertions(+), 39 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index a1f3a598170d18..cf979a6a4277ed 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -63,7 +63,7 @@ typedef struct _PyInterpreterFrame { PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */ _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */ - int stacktop; /* Offset of TOS from localsplus */ + PyObject **stackpointer; uint16_t return_offset; /* Only relevant during a function call */ char owner; /* Locals and stack */ @@ -83,20 +83,20 @@ static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) { } static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) { - assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - assert(f->localsplus[f->stacktop-1] != NULL); - return f->localsplus[f->stacktop-1]; + assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + assert(f->stackpointer[-1] != NULL); + return f->stackpointer[-1]; } static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) { - assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus); - f->stacktop--; - return f->localsplus[f->stacktop]; + assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); + f->stackpointer--; + return *f->stackpointer; } static inline void _PyFrame_StackPush(_PyInterpreterFrame *f, PyObject *value) { - f->localsplus[f->stacktop] = value; - f->stacktop++; + *f->stackpointer = value; + f->stackpointer++; } #define FRAME_SPECIALS_SIZE ((int)((sizeof(_PyInterpreterFrame)-1)/sizeof(PyObject *))) @@ -112,9 +112,12 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code) static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) { - assert(src->stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); *dest = *src; - for (int i = 1; i < src->stacktop; i++) { + assert(src->stackpointer != NULL); + int stacktop = (int)(src->stackpointer - src->localsplus); + assert(stacktop >= _PyFrame_GetCode(src)->co_nlocalsplus); + dest->stackpointer = dest->localsplus + stacktop; + for (int i = 1; i < stacktop; i++) { dest->localsplus[i] = src->localsplus[i]; } // Don't leave a dangling pointer to the old frame when creating generators @@ -136,7 +139,7 @@ _PyFrame_Initialize( frame->f_builtins = func->func_builtins; frame->f_globals = func->func_globals; frame->f_locals = locals; - frame->stacktop = code->co_nlocalsplus; + frame->stackpointer = frame->localsplus + code->co_nlocalsplus; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); frame->return_offset = 0; @@ -156,24 +159,23 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) return frame->localsplus; } -/* Fetches the stack pointer, and sets stacktop to -1. - Having stacktop <= 0 ensures that invalid - values are not visible to the cycle GC. - We choose -1 rather than 0 to assist debugging. */ +/* Fetches the stack pointer, and sets stackpointer to NULL. + Having stackpointer == NULL ensures that invalid + values are not visible to the cycle GC. */ static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - assert(frame->stacktop >= 0); - PyObject **sp = frame->localsplus + frame->stacktop; - frame->stacktop = -1; + assert(frame->stackpointer != NULL); + PyObject **sp = frame->stackpointer; + frame->stackpointer = NULL; return sp; } static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) { - assert(frame->stacktop == -1); - frame->stacktop = (int)(stack_pointer - frame->localsplus); + assert(frame->stackpointer == NULL); + frame->stackpointer = stack_pointer; } /* Determine whether a frame is incomplete. @@ -301,7 +303,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->f_globals = NULL; #endif frame->f_locals = NULL; - frame->stacktop = code->co_nlocalsplus + stackdepth; + assert(stackdepth <= code->co_stacksize); + frame->stackpointer = frame->localsplus + code->co_nlocalsplus + stackdepth; frame->frame_obj = NULL; frame->instr_ptr = _PyCode_CODE(code); frame->owner = FRAME_OWNED_BY_THREAD; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index fc8d6c7a7aee89..1d8727167f68e7 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1619,8 +1619,10 @@ frame_dealloc(PyFrameObject *f) Py_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); PyObject **locals = _PyFrame_GetLocalsArray(frame); - for (int i = 0; i < frame->stacktop; i++) { - Py_CLEAR(locals[i]); + PyObject **sp = frame->stackpointer; + while (sp > locals) { + sp--; + Py_CLEAR(*sp); } } Py_CLEAR(f->f_back); @@ -1652,11 +1654,13 @@ frame_tp_clear(PyFrameObject *f) /* locals and stack */ PyObject **locals = _PyFrame_GetLocalsArray(f->f_frame); - assert(f->f_frame->stacktop >= 0); - for (int i = 0; i < f->f_frame->stacktop; i++) { - Py_CLEAR(locals[i]); + PyObject **sp = f->f_frame->stackpointer; + assert(sp >= locals); + while (sp > locals) { + sp--; + Py_CLEAR(*sp); } - f->f_frame->stacktop = 0; + f->f_frame->stackpointer = locals; Py_CLEAR(f->f_frame->f_locals); return 0; } @@ -1874,7 +1878,7 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, } PyObject *value = frame->localsplus[i]; - if (frame->stacktop) { + if (frame->stackpointer > frame->localsplus) { if (kind & CO_FAST_FREE) { // The cell was set by COPY_FREE_VARS. assert(value != NULL && PyCell_Check(value)); diff --git a/Python/ceval.c b/Python/ceval.c index 324d062fe9bb43..89db590fddbc5c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -719,7 +719,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif entry_frame.f_executable = Py_None; entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; - entry_frame.stacktop = 0; + entry_frame.stackpointer = entry_frame.localsplus; entry_frame.owner = FRAME_OWNED_BY_CSTACK; entry_frame.return_offset = 0; /* Push frame */ diff --git a/Python/frame.c b/Python/frame.c index 2bb12823572028..32b4bf8ecc9c04 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -17,10 +17,11 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) Py_VISIT(_PyFrame_GetCode(frame)); /* locals */ PyObject **locals = _PyFrame_GetLocalsArray(frame); - int i = 0; + PyObject **sp = frame->stackpointer; /* locals and stack */ - for (; i stacktop; i++) { - Py_VISIT(locals[i]); + while (sp > locals) { + sp--; + Py_VISIT(*sp); } return 0; } @@ -59,10 +60,11 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) assert(frame->owner != FRAME_OWNED_BY_CSTACK); assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); assert(frame->owner != FRAME_CLEARED); - Py_ssize_t size = ((char*)&frame->localsplus[frame->stacktop]) - (char *)frame; + Py_ssize_t size = ((char*)frame->stackpointer) - (char *)frame; Py_INCREF(_PyFrame_GetCode(frame)); memcpy((_PyInterpreterFrame *)f->_f_frame_data, frame, size); frame = (_PyInterpreterFrame *)f->_f_frame_data; + frame->stackpointer = (PyObject **)(((char *)frame) + size); f->f_frame = frame; frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; if (_PyFrame_IsIncomplete(frame)) { @@ -97,11 +99,13 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) void _PyFrame_ClearLocals(_PyInterpreterFrame *frame) { - assert(frame->stacktop >= 0); - int stacktop = frame->stacktop; - frame->stacktop = 0; - for (int i = 0; i < stacktop; i++) { - Py_XDECREF(frame->localsplus[i]); + assert(frame->stackpointer != NULL); + PyObject **sp = frame->stackpointer; + PyObject **locals = frame->localsplus; + frame->stackpointer = locals; + while (sp > locals) { + sp--; + Py_XDECREF(*sp); } Py_CLEAR(frame->f_locals); } From 0a5086080c3b09a020e6ff552ff15add57b1541f Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 May 2024 17:32:27 +0100 Subject: [PATCH 4/7] Remove deferred RC stats --- Include/cpython/pystate.h | 1 + Include/cpython/pystats.h | 19 ------------------- Include/internal/pycore_code.h | 1 - Include/internal/pycore_frame.h | 6 ++++++ Include/pystats.h | 1 - Lib/test/test_sys.py | 4 ++-- Objects/object.c | 1 + Python/bytecodes.c | 1 - Python/ceval_macros.h | 3 +-- Python/executor_cases.c.h | 1 - Python/generated_cases.c.h | 1 - Python/pystate.c | 1 + Python/specialize.c | 4 ---- 13 files changed, 12 insertions(+), 32 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 387e2e7a9a138f..3efec47f3f4fcf 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -191,6 +191,7 @@ struct _ts { PyObject *previous_executor; uint64_t dict_global_version; + int sp_cached; /* Only used in debug builds */ }; diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index adfd45da574615..38480a4f6cd78f 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -87,12 +87,8 @@ typedef struct _object_stats { uint64_t type_cache_dunder_hits; uint64_t type_cache_dunder_misses; uint64_t type_cache_collisions; - uint64_t max_deferred_space; - uint64_t max_deferred_objects; /* Temporary value used during GC */ uint64_t object_visits; - uint64_t deferred_space; - uint64_t deferred_objects; } ObjectStats; typedef struct _gc_stats { @@ -171,18 +167,3 @@ PyAPI_DATA(PyStats*) _Py_stats; # define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) # define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) #endif - - -#define UPDATE_DEFERRED_STATS() \ -do { \ - if (_Py_stats) { \ - if (_Py_stats->object_stats.deferred_space > _Py_stats->object_stats.max_deferred_space) { \ - _Py_stats->object_stats.max_deferred_space = _Py_stats->object_stats.deferred_space; \ - } \ - if (_Py_stats->object_stats.deferred_objects > _Py_stats->object_stats.max_deferred_objects) { \ - _Py_stats->object_stats.max_deferred_objects = _Py_stats->object_stats.deferred_objects; \ - } \ - _Py_stats->object_stats.deferred_space = 0; \ - _Py_stats->object_stats.deferred_objects = 0; \ - } \ -} while (0) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 6be21000aa2387..ba23c645a8acba 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -356,7 +356,6 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OPT_ERROR_IN_OPCODE(opname) ((void)0) #define OPT_HIST(length, name) ((void)0) #define RARE_EVENT_STAT_INC(name) ((void)0) -#define UPDATE_DEFERRED_STATS() ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index cf979a6a4277ed..4aebf4a3449cd2 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -165,6 +165,9 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { +#ifndef Py_DEBUG + PyThreadState_GET()->sp_cached++; +#endif assert(frame->stackpointer != NULL); PyObject **sp = frame->stackpointer; frame->stackpointer = NULL; @@ -174,6 +177,9 @@ _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) { +#ifndef Py_DEBUG + PyThreadState_GET()->sp_cached--; +#endif assert(frame->stackpointer == NULL); frame->stackpointer = stack_pointer; } diff --git a/Include/pystats.h b/Include/pystats.h index 98dc5022e4620d..acfa32201711e0 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -18,7 +18,6 @@ extern "C" { #else # define _Py_INCREF_STAT_INC() ((void)0) # define _Py_DECREF_STAT_INC() ((void)0) -# define UPDATE_DEFERRED_STATS() ((void)0) #endif // !Py_STATS #ifdef __cplusplus diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 8fe1d77756866a..0562be2a9a55bb 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1561,7 +1561,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi2cP7P2ic??2P')) + check(x, size('3Pi2cP9Phc2P')) # function def func(): pass check(func, size('16Pi')) @@ -1578,7 +1578,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('PP4P4c7P2ic??2P')) + check(get_gen(), size('PP4P4cP9PhcP')) # iterator check(iter('abc'), size('lP')) # callable-iterator diff --git a/Objects/object.c b/Objects/object.c index 6fbc608958fff2..206aa491b5722a 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2829,6 +2829,7 @@ _Py_Dealloc(PyObject *op) destructor dealloc = type->tp_dealloc; #ifdef Py_DEBUG PyThreadState *tstate = _PyThreadState_GET(); + assert(tstate->sp_cached <= 1); PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL; // Keep the old exception type alive to prevent undefined behavior // on (tstate->curexc_type != old_exc_type) below diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 39ee6d595dfe5e..2aa3e5880ae55f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -184,7 +184,6 @@ dummy_func( uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - UPDATE_DEFERRED_STATS(); DEOPT_IF(eval_breaker != version); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index e896279c099f4f..90d10ddbf507a9 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -143,8 +143,7 @@ do { \ if (err != 0) { \ GOTO_ERROR(error); \ } \ - } \ - UPDATE_DEFERRED_STATS(); + } /* Tuple access macros */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6e5f1cca9471f8..f2dda0cbad7da2 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -23,7 +23,6 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - UPDATE_DEFERRED_STATS(); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cc2cec10dc517e..8a1d4080f5edd0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5263,7 +5263,6 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); - UPDATE_DEFERRED_STATS(); DEOPT_IF(eval_breaker != version, RESUME); DISPATCH(); } diff --git a/Python/pystate.c b/Python/pystate.c index 1ea1ad982a0ec9..d047bf8f6dcc5b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1498,6 +1498,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->what_event = -1; tstate->previous_executor = NULL; tstate->dict_global_version = 0; + tstate->sp_cached = 0; tstate->delete_later = NULL; diff --git a/Python/specialize.c b/Python/specialize.c index 23b91f8713e646..9ac428c3593f56 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -203,8 +203,6 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); - fprintf(out, "Object max deferred objects: %" PRIu64 "\n", stats->max_deferred_objects); - fprintf(out, "Object max deferred space: %" PRIu64 "\n", stats->max_deferred_space); } static void @@ -321,8 +319,6 @@ void _Py_StatsOn(void) { _Py_stats = &_Py_stats_struct; - _Py_stats->object_stats.deferred_objects = 0; - _Py_stats->object_stats.deferred_space = 0; } void From c2775e0c4b066471bf98faa0c9ea6c0ca8b48448 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 May 2024 17:58:30 +0100 Subject: [PATCH 5/7] Remove OBJECT_STAT_ADD --- Include/internal/pycore_code.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index ba23c645a8acba..bcbaf60f226c77 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -307,7 +307,6 @@ extern void _Py_Specialize_ContainsOp(PyObject *value, _Py_CODEUNIT *instr); #define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0) #define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0) #define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0) -#define OBJECT_STAT_ADD(name, value) do { if (_Py_stats) _Py_stats->object_stats.name += value; } while (0) #define OBJECT_STAT_INC_COND(name, cond) \ do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0) #define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0) @@ -344,7 +343,6 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OPCODE_EXE_INC(opname) ((void)0) #define CALL_STAT_INC(name) ((void)0) #define OBJECT_STAT_INC(name) ((void)0) -#define OBJECT_STAT_ADD(name, val) ((void)0) #define OBJECT_STAT_INC_COND(name, cond) ((void)0) #define EVAL_CALL_STAT_INC(name) ((void)0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) From 2bd13390b05ee25fb2455bd4050e0c7ccd437ba3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 31 May 2024 18:33:00 +0100 Subject: [PATCH 6/7] Restore whitespace --- Objects/object.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/object.c b/Objects/object.c index 206aa491b5722a..6e8a9da2418a76 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2822,6 +2822,8 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, Py_FatalError("_PyObject_AssertFailed"); } + + void _Py_Dealloc(PyObject *op) { @@ -2847,7 +2849,6 @@ _Py_Dealloc(PyObject *op) #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif - (*dealloc)(op); #ifdef Py_DEBUG From 792c553057bd586806ef2d788171dbb81fff813b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 6 Jun 2024 10:45:57 +0100 Subject: [PATCH 7/7] Add many more SAVE_SP/LOAD_SP pairs --- Include/internal/pycore_frame.h | 4 +- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_metadata.h | 2 +- Objects/object.c | 4 +- Python/bytecodes.c | 289 ++++++++- Python/ceval.c | 69 ++- Python/ceval_macros.h | 11 +- Python/executor_cases.c.h | 490 +++++++++------ Python/generated_cases.c.h | 683 ++++++++++++++------- Tools/cases_generator/generators_common.py | 34 +- 10 files changed, 1140 insertions(+), 448 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 4aebf4a3449cd2..81590dde93ac06 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -165,7 +165,7 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { -#ifndef Py_DEBUG +#ifdef Py_DEBUG PyThreadState_GET()->sp_cached++; #endif assert(frame->stackpointer != NULL); @@ -177,7 +177,7 @@ _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer) { -#ifndef Py_DEBUG +#ifdef Py_DEBUG PyThreadState_GET()->sp_cached--; #endif assert(frame->stackpointer == NULL); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d3535800139a66..c39c4591ac5b98 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1149,7 +1149,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [POP_EXCEPT] = { true, INSTR_FMT_IX, 0 }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 78f0eafaa32042..f73a581b4d25b3 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -92,7 +92,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_POP_EXCEPT] = HAS_ESCAPES_FLAG, + [_POP_EXCEPT] = 0, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Objects/object.c b/Objects/object.c index 6e8a9da2418a76..a9edc72deca91d 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2831,7 +2831,9 @@ _Py_Dealloc(PyObject *op) destructor dealloc = type->tp_dealloc; #ifdef Py_DEBUG PyThreadState *tstate = _PyThreadState_GET(); - assert(tstate->sp_cached <= 1); +#ifndef Py_GIL_DISABLED + assert(tstate->sp_cached == 0); +#endif PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL; // Keep the old exception type alive to prevent undefined behavior // on (tstate->curexc_type != old_exc_type) below diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2aa3e5880ae55f..d2a38056ae5ef6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -344,7 +344,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ToBool(value, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); @@ -420,7 +422,9 @@ dummy_func( _REPLACE_WITH_TRUE; inst(UNARY_INVERT, (value -- res)) { + SAVE_SP(); res = PyNumber_Invert(value); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } @@ -564,7 +568,9 @@ dummy_func( */ assert(Py_REFCNT(left) >= 2); _Py_DECREF_NO_DEALLOC(left); + SAVE_SP(); PyUnicode_Append(target_local, right); + LOAD_SP(); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); ERROR_IF(*target_local == NULL, error); // The STORE_FAST is already done. @@ -587,7 +593,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_BinarySubscr(container, sub, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); @@ -615,7 +623,9 @@ dummy_func( res = NULL; } else { + SAVE_SP(); res = PyObject_GetItem(container, slice); + LOAD_SP(); Py_DECREF(slice); } Py_DECREF(container); @@ -689,7 +699,9 @@ dummy_func( inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) { DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(BINARY_SUBSCR, hit); + SAVE_SP(); int rc = PyDict_GetItemRef(dict, sub, &res); + LOAD_SP(); if (rc == 0) { _PyErr_SetKeyError(sub); } @@ -726,7 +738,9 @@ dummy_func( } inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) { + SAVE_SP(); int err = PySet_Add(set, v); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -740,7 +754,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_StoreSubscr(container, sub, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); @@ -781,7 +797,9 @@ dummy_func( inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); + SAVE_SP(); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); + LOAD_SP(); Py_DECREF(dict); ERROR_IF(err, error); } @@ -815,6 +833,7 @@ dummy_func( tier1 inst(RAISE_VARARGS, (args[oparg] -- )) { PyObject *cause = NULL, *exc = NULL; + SAVE_SP(); switch (oparg) { case 2: cause = args[1]; @@ -826,6 +845,7 @@ dummy_func( if (do_raise(tstate, exc, cause)) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } break; @@ -834,6 +854,7 @@ dummy_func( "bad RAISE_VARARGS oparg"); break; } + LOAD_SP(); ERROR_IF(true, error); } @@ -874,9 +895,11 @@ dummy_func( _POP_FRAME; inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { + SAVE_SP(); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); + LOAD_SP(); if (err) ERROR_NO_POP(); STACK_SHRINK(1); assert(EMPTY()); @@ -898,9 +921,11 @@ dummy_func( inst(INSTRUMENTED_RETURN_CONST, (--)) { PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + SAVE_SP(); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); + LOAD_SP(); if (err) ERROR_NO_POP(); Py_INCREF(retval); assert(EMPTY()); @@ -925,15 +950,19 @@ dummy_func( } if (getter == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(true, error); } + SAVE_SP(); iter = (*getter)(obj); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(iter == NULL, error); @@ -955,7 +984,9 @@ dummy_func( PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { + SAVE_SP(); awaitable = type->tp_as_async->am_anext(aiter); + LOAD_SP(); if (awaitable == NULL) { ERROR_NO_POP(); } @@ -965,27 +996,34 @@ dummy_func( } if (getter != NULL) { + SAVE_SP(); next_iter = (*getter)(aiter); + LOAD_SP(); if (next_iter == NULL) { ERROR_NO_POP(); } } else { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); + LOAD_SP(); ERROR_NO_POP(); } + SAVE_SP(); awaitable = _PyCoro_GetAwaitableIter(next_iter); + LOAD_SP(); if (awaitable == NULL) { + SAVE_SP(); _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - + LOAD_SP(); Py_DECREF(next_iter); ERROR_NO_POP(); } else { @@ -995,10 +1033,14 @@ dummy_func( } inst(GET_AWAITABLE, (iterable -- iter)) { + SAVE_SP(); iter = _PyCoro_GetAwaitableIter(iterable); + LOAD_SP(); if (iter == NULL) { + SAVE_SP(); _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + LOAD_SP(); } DECREF_INPUTS(); @@ -1028,7 +1070,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_Send(receiver, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); @@ -1064,9 +1108,14 @@ dummy_func( if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { + SAVE_SP(); monitor_raise(tstate, frame, this_instr); + LOAD_SP(); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { + SAVE_SP(); + int err = _PyGen_FetchStopIterationValue(&retval); + LOAD_SP(); + if (err == 0) { assert(retval != NULL); JUMPBY(oparg); } @@ -1159,7 +1208,11 @@ dummy_func( inst(POP_EXCEPT, (exc_value -- )) { _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + PyObject *tmp = exc_info->exc_value; + exc_info->exc_value = exc_value == Py_None ? NULL : exc_value; + if (tmp != NULL) { + INTERPRETER_DECREF(tmp); + } } tier1 inst(RERAISE, (values[oparg], exc -- values[oparg])) { @@ -1179,7 +1232,9 @@ dummy_func( assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } @@ -1191,7 +1246,9 @@ dummy_func( else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } } @@ -1206,7 +1263,9 @@ dummy_func( } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } } @@ -1226,7 +1285,10 @@ dummy_func( } inst(LOAD_BUILD_CLASS, ( -- bc)) { - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0, error); + SAVE_SP(); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc); + LOAD_SP(); + ERROR_IF(err < 0, error); if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); @@ -1259,16 +1321,22 @@ dummy_func( PyObject *ns = LOCALS(); int err; if (ns == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); + LOAD_SP(); ERROR_NO_POP(); } + SAVE_SP(); err = PyObject_DelItem(ns, name); + LOAD_SP(); // Can't use ERROR_IF here. if (err != 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); ERROR_NO_POP(); } } @@ -1283,7 +1351,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); @@ -1295,7 +1365,9 @@ dummy_func( op(_UNPACK_SEQUENCE, (seq -- unused[oparg])) { PyObject **top = stack_pointer + oparg - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1337,7 +1409,9 @@ dummy_func( inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) { int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res == 0, error); } @@ -1353,7 +1427,9 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_StoreAttr(owner, next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); @@ -1374,28 +1450,36 @@ dummy_func( inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_DelAttr(owner, name); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } inst(STORE_GLOBAL, (v --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_SetItem(GLOBALS(), name, v); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err, error); } inst(DELETE_GLOBAL, (--)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_Pop(GLOBALS(), name, NULL); + LOAD_SP(); // Can't use ERROR_IF here. if (err < 0) { ERROR_NO_POP(); } if (err == 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); ERROR_NO_POP(); } } @@ -1403,8 +1487,10 @@ dummy_func( inst(LOAD_LOCALS, ( -- locals)) { locals = LOCALS(); if (locals == NULL) { + SAVE_SP(); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + LOAD_SP(); ERROR_IF(true, error); } Py_INCREF(locals); @@ -1412,21 +1498,32 @@ dummy_func( inst(LOAD_FROM_DICT_OR_GLOBALS, (mod_or_class_dict -- v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(mod_or_class_dict, name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + SAVE_SP(); + err = PyDict_GetItemRef(GLOBALS(), name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); ERROR_NO_POP(); } } @@ -1437,26 +1534,39 @@ dummy_func( inst(LOAD_NAME, (-- v)) { PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { + SAVE_SP(); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + LOAD_SP(); ERROR_IF(true, error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(mod_or_class_dict, name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + SAVE_SP(); + err = PyDict_GetItemRef(GLOBALS(), name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &v); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); ERROR_NO_POP(); } } @@ -1473,7 +1583,9 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); @@ -1486,15 +1598,19 @@ dummy_func( if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { + SAVE_SP(); res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); + LOAD_SP(); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); } ERROR_IF(true, error); } @@ -1502,14 +1618,22 @@ dummy_func( else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0, error); + SAVE_SP(); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &res); + LOAD_SP(); + ERROR_IF(err < 0, error); if (res == NULL) { /* namespace 2: builtins */ - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0, error); + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &res); + LOAD_SP(); + ERROR_IF(err < 0, error); if (res == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); ERROR_IF(true, error); } } @@ -1573,10 +1697,12 @@ dummy_func( inst(DELETE_FAST, (--)) { PyObject *v = GETLOCAL(oparg); if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + LOAD_SP(); ERROR_IF(1, error); } SETLOCAL(oparg, NULL); @@ -1599,7 +1725,9 @@ dummy_func( // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); ERROR_NO_POP(); } Py_DECREF(oldobj); @@ -1610,14 +1738,19 @@ dummy_func( assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(class_dict, name, &value); + LOAD_SP(); + if (err < 0) { ERROR_NO_POP(); } if (!value) { PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); ERROR_NO_POP(); } } @@ -1628,14 +1761,18 @@ dummy_func( PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); ERROR_IF(true, error); } } inst(STORE_DEREF, (v --)) { PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + SAVE_SP(); PyCell_SetTakeRef(cell, v); + LOAD_SP(); } inst(COPY_FREE_VARS, (--)) { @@ -1668,8 +1805,11 @@ dummy_func( } inst(LIST_EXTEND, (list, unused[oparg-1], iterable -- list, unused[oparg-1])) { + SAVE_SP(); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + LOAD_SP(); if (none_val == NULL) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { @@ -1679,6 +1819,7 @@ dummy_func( Py_TYPE(iterable)->tp_name); } DECREF_INPUTS(); + LOAD_SP(); ERROR_IF(true, error); } assert(Py_IsNone(none_val)); @@ -1686,7 +1827,9 @@ dummy_func( } inst(SET_UPDATE, (set, unused[oparg-1], iterable -- set, unused[oparg-1])) { + SAVE_SP(); int err = _PySet_Update(set, iterable); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(err < 0, error); } @@ -1698,8 +1841,11 @@ dummy_func( int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; - if (err == 0) + if (err == 0) { + SAVE_SP(); err = PySet_Add(set, item); + LOAD_SP(); + } Py_DECREF(item); } if (err != 0) { @@ -1709,10 +1855,12 @@ dummy_func( } inst(BUILD_MAP, (values[oparg*2] -- map)) { + SAVE_SP(); map = _PyDict_FromItems( values, 2, values+1, 2, oparg); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(map == NULL, error); } @@ -1721,12 +1869,17 @@ dummy_func( int err; PyObject *ann_dict; if (LOCALS() == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); + LOAD_SP(); ERROR_IF(true, error); } /* check if __annotations__ in locals()... */ - ERROR_IF(PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0, error); + SAVE_SP(); + err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + LOAD_SP(); + ERROR_IF(err < 0, error); if (ann_dict == NULL) { ann_dict = PyDict_New(); ERROR_IF(ann_dict == NULL, error); @@ -1743,9 +1896,11 @@ dummy_func( inst(BUILD_CONST_KEY_MAP, (values[oparg], keys -- map)) { assert(PyTuple_CheckExact(keys)); assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); + SAVE_SP(); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(map == NULL, error); } @@ -1755,11 +1910,13 @@ dummy_func( int err = PyDict_Update(dict, update); LOAD_SP(); if (err < 0) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(true, error); } @@ -1771,7 +1928,9 @@ dummy_func( int err = _PyDict_MergeEx(dict, update, 2); LOAD_SP(); if (err < 0) { + SAVE_SP(); _PyEval_FormatKwargsError(tstate, callable, update); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(true, error); } @@ -1782,7 +1941,10 @@ dummy_func( assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); + SAVE_SP(); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, key, value); + LOAD_SP(); + ERROR_IF(err != 0, error); } inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) { @@ -1802,7 +1964,9 @@ dummy_func( int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); @@ -1812,15 +1976,18 @@ dummy_func( tier1 op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) { if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + SAVE_SP(); PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); + LOAD_SP(); ERROR_IF(err, error); } // we make no attempt to optimize here; specializations should // handle any case whose performance we care about PyObject *stack[] = {class, self}; + SAVE_SP(); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; @@ -1838,10 +2005,13 @@ dummy_func( } } } + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(super == NULL, error); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + SAVE_SP(); attr = PyObject_GetAttr(super, name); + LOAD_SP(); Py_DECREF(super); ERROR_IF(attr == NULL, error); null = NULL; @@ -1855,7 +2025,9 @@ dummy_func( DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + SAVE_SP(); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(attr == NULL, error); } @@ -1868,8 +2040,10 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; + SAVE_SP(); attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + LOAD_SP(); Py_DECREF(global_super); Py_DECREF(class); if (attr == NULL) { @@ -1904,7 +2078,9 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadAttr(owner, next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); @@ -2207,7 +2383,9 @@ dummy_func( STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; - Py_XDECREF(old_value); + if (old_value != NULL) { + Py_DECREF(old_value); + } Py_DECREF(owner); } @@ -2226,7 +2404,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_CompareOp(left, right, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); @@ -2327,7 +2507,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ContainsOp(right, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(CONTAINS_OP, deferred); @@ -2341,7 +2523,9 @@ dummy_func( DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! + SAVE_SP(); int res = _PySet_Contains((PySetObject *)right, left); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res < 0, error); b = (res ^ oparg) ? Py_True : Py_False; @@ -2350,7 +2534,9 @@ dummy_func( inst(CONTAINS_OP_DICT, (unused/1, left, right -- b)) { DEOPT_IF(!PyDict_CheckExact(right)); STAT_INC(CONTAINS_OP, hit); + SAVE_SP(); int res = PyDict_Contains(right, left); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res < 0, error); b = (res ^ oparg) ? Py_True : Py_False; @@ -2364,8 +2550,10 @@ dummy_func( match = NULL; rest = NULL; + SAVE_SP(); int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, &match, &rest); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(res < 0, error); @@ -2379,12 +2567,14 @@ dummy_func( inst(CHECK_EXC_MATCH, (left, right -- left, b)) { assert(PyExceptionInstance_Check(left)); + SAVE_SP(); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(true, error); } - int res = PyErr_GivenExceptionMatches(left, right); + LOAD_SP(); DECREF_INPUTS(); b = res ? Py_True : Py_False; } @@ -2538,7 +2728,9 @@ dummy_func( // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); + SAVE_SP(); attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + LOAD_SP(); DECREF_INPUTS(); if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! @@ -2561,13 +2753,17 @@ dummy_func( inst(MATCH_KEYS, (subject, keys -- subject, keys, values_or_none)) { // On successful match, PUSH(values). Otherwise, PUSH(None). + SAVE_SP(); values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + LOAD_SP(); ERROR_IF(values_or_none == NULL, error); } inst(GET_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ + SAVE_SP(); iter = PyObject_GetIter(iterable); + LOAD_SP(); DECREF_INPUTS(); ERROR_IF(iter == NULL, error); } @@ -2591,7 +2787,9 @@ dummy_func( } else { /* `iterable` is not a generator. */ + SAVE_SP(); iter = PyObject_GetIter(iterable); + LOAD_SP(); if (iter == NULL) { ERROR_NO_POP(); } @@ -2616,7 +2814,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ForIter(iter, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); @@ -2634,8 +2834,10 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { ERROR_NO_POP(); } + SAVE_SP(); monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || @@ -2651,13 +2853,17 @@ dummy_func( op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + SAVE_SP(); next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { ERROR_NO_POP(); } + SAVE_SP(); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ /* The translator sets the deopt target just past the matching END_FOR */ @@ -2671,7 +2877,9 @@ dummy_func( inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { _Py_CODEUNIT *target; PyObject *iter = TOP(); + SAVE_SP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next != NULL) { PUSH(next); target = next_instr; @@ -2681,8 +2889,10 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { ERROR_NO_POP(); } + SAVE_SP(); monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || @@ -3098,9 +3308,11 @@ dummy_func( PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args); + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + LOAD_SP(); ERROR_IF(err, error); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL); @@ -3135,7 +3347,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); @@ -3168,10 +3382,12 @@ dummy_func( { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); // The frame has stolen all the arguments from the stack, @@ -3192,6 +3408,7 @@ dummy_func( if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; + SAVE_SP(); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -3205,6 +3422,7 @@ dummy_func( Py_CLEAR(res); } } + LOAD_SP(); } assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(callable); @@ -3230,10 +3448,12 @@ dummy_func( assert(Py_TYPE(callable) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. SYNC_SP(); @@ -3861,9 +4081,11 @@ dummy_func( PyObject *function = PEEK(oparg + 3); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args + 1); + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + LOAD_SP(); ERROR_IF(err, error); GO_TO_INSTRUCTION(CALL_KW); } @@ -3893,10 +4115,12 @@ dummy_func( { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, positional_args, kwnames ); + LOAD_SP(); Py_DECREF(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 3); @@ -3919,6 +4143,7 @@ dummy_func( if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; + SAVE_SP(); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -3932,6 +4157,7 @@ dummy_func( Py_CLEAR(res); } } + LOAD_SP(); } Py_DECREF(kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -3955,33 +4181,43 @@ dummy_func( if (check_args_iterable(tstate, func, callargs) < 0) { ERROR_NO_POP(); } + SAVE_SP(); PyObject *tuple = PySequence_Tuple(callargs); + LOAD_SP(); if (tuple == NULL) { ERROR_NO_POP(); } - Py_SETREF(callargs, tuple); + Py_DECREF(callargs); + callargs = tuple; } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); + LOAD_SP(); if (err) ERROR_NO_POP(); + SAVE_SP(); result = PyObject_Call(func, callargs, kwargs); - + LOAD_SP(); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result == NULL) { + SAVE_SP(); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); + LOAD_SP(); } else { + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); + LOAD_SP(); if (err < 0) { Py_CLEAR(result); } @@ -3996,10 +4232,11 @@ dummy_func( Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); + LOAD_SP(); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -4020,10 +4257,10 @@ dummy_func( } inst(MAKE_FUNCTION, (codeobj -- func)) { - + SAVE_SP(); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - + LOAD_SP(); Py_DECREF(codeobj); if (func_obj == NULL) { ERROR_NO_POP(); @@ -4107,7 +4344,9 @@ dummy_func( /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { + SAVE_SP(); res = PyObject_Format(value, NULL); + LOAD_SP(); Py_DECREF(value); ERROR_IF(res == NULL, error); } @@ -4117,7 +4356,9 @@ dummy_func( } inst(FORMAT_WITH_SPEC, (value, fmt_spec -- res)) { + SAVE_SP(); res = PyObject_Format(value, fmt_spec); + LOAD_SP(); Py_DECREF(value); Py_DECREF(fmt_spec); ERROR_IF(res == NULL, error); @@ -4132,7 +4373,9 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); diff --git a/Python/ceval.c b/Python/ceval.c index 89db590fddbc5c..b16ebff4ee6e9a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -51,16 +51,34 @@ # error "ceval.c must be build with Py_BUILD_CORE define for best performance" #endif -#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_GIL_DISABLED) -// GH-89279: The MSVC compiler does not inline these static inline functions -// in PGO build in _PyEval_EvalFrameDefault(), because this function is over -// the limit of PGO, and that limit cannot be configured. -// Define them as macros to make sure that they are always inlined by the -// preprocessor. -// TODO: implement Py_DECREF macro for Py_GIL_DISABLED - -#undef Py_DECREF -#define Py_DECREF(arg) \ +#ifdef Py_GIL_DISABLED + +#define INTERPRETER_DECREF Py_DECREF + +#else + +#ifdef Py_REF_DEBUG +static inline void interpreter_decref(_PyInterpreterFrame *frame, PyObject **sp, const char *filename, int lineno, PyObject *op) +{ + if (op->ob_refcnt <= 0) { + _Py_NegativeRefcount(filename, lineno, op); + } + if (_Py_IsImmortal(op)) { + return; + } + _Py_DECREF_STAT_INC(); + _Py_DECREF_DecRefTotal(); + if (--op->ob_refcnt == 0) { + _PyFrame_SetStackPointer(frame, sp); + _Py_Dealloc(op); + _PyFrame_GetStackPointer(frame); + } +} +#define INTERPRETER_DECREF(op) interpreter_decref(frame, stack_pointer, __FILE__, __LINE__, _PyObject_CAST(op)) + +#else + +#define INTERPRETER_DECREF(arg) \ do { \ PyObject *op = _PyObject_CAST(arg); \ if (_Py_IsImmortal(op)) { \ @@ -68,19 +86,14 @@ } \ _Py_DECREF_STAT_INC(); \ if (--op->ob_refcnt == 0) { \ - destructor dealloc = Py_TYPE(op)->tp_dealloc; \ - (*dealloc)(op); \ + _PyFrame_SetStackPointer(frame, stack_pointer) \ + _Py_Dealloc(op); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ } \ } while (0) +#endif // Py_REF_DEBUG -#undef Py_XDECREF -#define Py_XDECREF(arg) \ - do { \ - PyObject *xop = _PyObject_CAST(arg); \ - if (xop != NULL) { \ - Py_DECREF(xop); \ - } \ - } while (0) +#endif // Py_GIL_DISABLED #undef Py_IS_TYPE #define Py_IS_TYPE(ob, type) \ @@ -99,7 +112,6 @@ d(op); \ } \ } while (0) -#endif #ifdef LLTRACE @@ -882,7 +894,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyTraceBack_Here(f); } } + SAVE_SP(); monitor_raise(tstate, frame, next_instr-1); + LOAD_SP(); exception_unwind: { /* We can't use frame->instr_ptr here, as RERAISE may have set it */ @@ -896,7 +910,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject **stackbase = _PyFrame_Stackbase(frame); while (stack_pointer > stackbase) { PyObject *o = POP(); - Py_XDECREF(o); + if (o != NULL) { + INTERPRETER_DECREF(o); + } } assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -908,7 +924,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject **new_top = _PyFrame_Stackbase(frame) + level; while (stack_pointer > new_top) { PyObject *v = POP(); - Py_XDECREF(v); + if (v != NULL) { + INTERPRETER_DECREF(v); + } } if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); @@ -927,7 +945,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PUSH(exc); next_instr = _PyCode_CODE(_PyFrame_GetCode(frame)) + handler; - if (monitor_handled(tstate, frame, next_instr, exc) < 0) { + SAVE_SP(); + int err = monitor_handled(tstate, frame, next_instr, exc); + LOAD_SP(); + if (err < 0) { goto exception_unwind; } /* Resume normal execution */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 90d10ddbf507a9..9828162f6ada36 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -265,9 +265,14 @@ GETITEM(PyObject *v, Py_ssize_t i) { This is because it is possible that during the DECREF the frame is accessed by other code (e.g. a __del__ method or gc.collect()) and the variable would be pointing to already-freed memory. */ -#define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \ - GETLOCAL(i) = value; \ - Py_XDECREF(tmp); } while (0) +#define SETLOCAL(i, value) \ +do { \ + PyObject *tmp = frame->localsplus[i]; \ + frame->localsplus[i] = value; \ + if (tmp != NULL) { \ + INTERPRETER_DECREF(tmp); \ + } \ +} while (0) #define GO_TO_INSTRUCTION(op) goto PREDICT_ID(op) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f2dda0cbad7da2..4961fc02b7ff2e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -269,7 +269,7 @@ case _POP_TOP: { PyObject *value; value = stack_pointer[-1]; - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer += -1; break; } @@ -287,7 +287,7 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); + INTERPRETER_DECREF(receiver); stack_pointer[-2] = value; stack_pointer += -1; break; @@ -300,7 +300,7 @@ SAVE_SP(); res = PyNumber_Negative(value); LOAD_SP(); - Py_DECREF(value); + INTERPRETER_DECREF(value); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; @@ -323,7 +323,7 @@ SAVE_SP(); int err = PyObject_IsTrue(value); LOAD_SP(); - Py_DECREF(value); + INTERPRETER_DECREF(value); if (err < 0) JUMP_TO_ERROR(); res = err ? Py_True : Py_False; stack_pointer[-1] = res; @@ -355,7 +355,7 @@ res = Py_False; } else { - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; } stack_pointer[-1] = res; @@ -372,7 +372,7 @@ } STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer[-1] = res; break; } @@ -407,7 +407,7 @@ } else { assert(Py_SIZE(value)); - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; } stack_pointer[-1] = res; @@ -418,7 +418,7 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; stack_pointer[-1] = res; break; @@ -428,8 +428,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + SAVE_SP(); res = PyNumber_Invert(value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; @@ -644,8 +646,8 @@ SAVE_SP(); res = PyObject_GetItem(container, sub); LOAD_SP(); - Py_DECREF(container); - Py_DECREF(sub); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; @@ -669,10 +671,12 @@ res = NULL; } else { + SAVE_SP(); res = PyObject_GetItem(container, slice); - Py_DECREF(slice); + LOAD_SP(); + INTERPRETER_DECREF(slice); } - Py_DECREF(container); + INTERPRETER_DECREF(container); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; @@ -697,10 +701,10 @@ SAVE_SP(); err = PyObject_SetItem(container, slice, v); LOAD_SP(); - Py_DECREF(slice); + INTERPRETER_DECREF(slice); } - Py_DECREF(v); - Py_DECREF(container); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(container); if (err) JUMP_TO_ERROR(); stack_pointer += -4; break; @@ -735,7 +739,7 @@ assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + INTERPRETER_DECREF(list); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -773,7 +777,7 @@ STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + INTERPRETER_DECREF(str); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -808,7 +812,7 @@ assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + INTERPRETER_DECREF(tuple); stack_pointer[-2] = res; stack_pointer += -1; break; @@ -825,12 +829,14 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_SUBSCR, hit); + SAVE_SP(); int rc = PyDict_GetItemRef(dict, sub, &res); + LOAD_SP(); if (rc == 0) { _PyErr_SetKeyError(sub); } - Py_DECREF(dict); - Py_DECREF(sub); + INTERPRETER_DECREF(dict); + INTERPRETER_DECREF(sub); if (rc <= 0) JUMP_TO_ERROR(); // not found or error stack_pointer[-2] = res; @@ -857,8 +863,10 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); int err = PySet_Add(set, v); - Py_DECREF(v); + LOAD_SP(); + INTERPRETER_DECREF(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; @@ -875,9 +883,9 @@ SAVE_SP(); int err = PyObject_SetItem(container, sub, v); LOAD_SP(); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (err) JUMP_TO_ERROR(); stack_pointer += -3; break; @@ -913,9 +921,9 @@ PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); assert(old_value != NULL); - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + INTERPRETER_DECREF(list); stack_pointer += -3; break; } @@ -932,8 +940,10 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(STORE_SUBSCR, hit); + SAVE_SP(); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); + LOAD_SP(); + INTERPRETER_DECREF(dict); if (err) JUMP_TO_ERROR(); stack_pointer += -3; break; @@ -948,8 +958,8 @@ SAVE_SP(); int err = PyObject_DelItem(container, sub); LOAD_SP(); - Py_DECREF(container); - Py_DECREF(sub); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (err) JUMP_TO_ERROR(); stack_pointer += -2; break; @@ -964,7 +974,7 @@ SAVE_SP(); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); LOAD_SP(); - Py_DECREF(value); + INTERPRETER_DECREF(value); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = res; break; @@ -981,8 +991,8 @@ SAVE_SP(); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); LOAD_SP(); - Py_DECREF(value2); - Py_DECREF(value1); + INTERPRETER_DECREF(value2); + INTERPRETER_DECREF(value1); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; @@ -1024,15 +1034,19 @@ getter = type->tp_as_async->am_aiter; } if (getter == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); + LOAD_SP(); + INTERPRETER_DECREF(obj); if (true) JUMP_TO_ERROR(); } + SAVE_SP(); iter = (*getter)(obj); - Py_DECREF(obj); + LOAD_SP(); + INTERPRETER_DECREF(obj); if (iter == NULL) JUMP_TO_ERROR(); if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { @@ -1040,7 +1054,7 @@ "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); - Py_DECREF(iter); + INTERPRETER_DECREF(iter); if (true) JUMP_TO_ERROR(); } stack_pointer[-1] = iter; @@ -1055,7 +1069,9 @@ PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { + SAVE_SP(); awaitable = type->tp_as_async->am_anext(aiter); + LOAD_SP(); if (awaitable == NULL) { JUMP_TO_ERROR(); } @@ -1064,29 +1080,37 @@ getter = type->tp_as_async->am_anext; } if (getter != NULL) { + SAVE_SP(); next_iter = (*getter)(aiter); + LOAD_SP(); if (next_iter == NULL) { JUMP_TO_ERROR(); } } else { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); + LOAD_SP(); JUMP_TO_ERROR(); } + SAVE_SP(); awaitable = _PyCoro_GetAwaitableIter(next_iter); + LOAD_SP(); if (awaitable == NULL) { + SAVE_SP(); _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); + LOAD_SP(); + INTERPRETER_DECREF(next_iter); JUMP_TO_ERROR(); } else { - Py_DECREF(next_iter); + INTERPRETER_DECREF(next_iter); } } stack_pointer[0] = awaitable; @@ -1099,18 +1123,22 @@ PyObject *iter; oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; + SAVE_SP(); iter = _PyCoro_GetAwaitableIter(iterable); + LOAD_SP(); if (iter == NULL) { + SAVE_SP(); _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + LOAD_SP(); } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ - Py_DECREF(yf); + INTERPRETER_DECREF(yf); Py_CLEAR(iter); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); @@ -1175,7 +1203,11 @@ PyObject *exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + PyObject *tmp = exc_info->exc_value; + exc_info->exc_value = exc_value == Py_None ? NULL : exc_value; + if (tmp != NULL) { + INTERPRETER_DECREF(tmp); + } stack_pointer += -1; break; } @@ -1201,7 +1233,10 @@ case _LOAD_BUILD_CLASS: { PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) JUMP_TO_ERROR(); + SAVE_SP(); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc); + LOAD_SP(); + if (err < 0) JUMP_TO_ERROR(); if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); @@ -1222,7 +1257,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); + INTERPRETER_DECREF(v); if (true) JUMP_TO_ERROR(); } SAVE_SP(); @@ -1231,7 +1266,7 @@ else err = PyObject_SetItem(ns, name, v); LOAD_SP(); - Py_DECREF(v); + INTERPRETER_DECREF(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; @@ -1243,16 +1278,22 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); + LOAD_SP(); JUMP_TO_ERROR(); } + SAVE_SP(); err = PyObject_DelItem(ns, name); + LOAD_SP(); // Can't use ERROR_IF here. if (err != 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); JUMP_TO_ERROR(); } break; @@ -1263,8 +1304,10 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; PyObject **top = stack_pointer + oparg - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); + LOAD_SP(); + INTERPRETER_DECREF(seq); if (res == 0) JUMP_TO_ERROR(); stack_pointer += -1 + oparg; break; @@ -1288,7 +1331,7 @@ STAT_INC(UNPACK_SEQUENCE, hit); val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -1314,7 +1357,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer += -1 + oparg; break; } @@ -1338,7 +1381,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer += -1 + oparg; break; } @@ -1349,8 +1392,10 @@ seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); + LOAD_SP(); + INTERPRETER_DECREF(seq); if (res == 0) JUMP_TO_ERROR(); stack_pointer += (oparg >> 8) + (oparg & 0xFF); break; @@ -1366,8 +1411,8 @@ SAVE_SP(); int err = PyObject_SetAttr(owner, name, v); LOAD_SP(); - Py_DECREF(v); - Py_DECREF(owner); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(owner); if (err) JUMP_TO_ERROR(); stack_pointer += -2; break; @@ -1378,8 +1423,10 @@ oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); + LOAD_SP(); + INTERPRETER_DECREF(owner); if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; @@ -1390,8 +1437,10 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); + LOAD_SP(); + INTERPRETER_DECREF(v); if (err) JUMP_TO_ERROR(); stack_pointer += -1; break; @@ -1400,14 +1449,18 @@ case _DELETE_GLOBAL: { oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_Pop(GLOBALS(), name, NULL); + LOAD_SP(); // Can't use ERROR_IF here. if (err < 0) { JUMP_TO_ERROR(); } if (err == 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); JUMP_TO_ERROR(); } break; @@ -1417,8 +1470,10 @@ PyObject *locals; locals = LOCALS(); if (locals == NULL) { + SAVE_SP(); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + LOAD_SP(); if (true) JUMP_TO_ERROR(); } Py_INCREF(locals); @@ -1433,26 +1488,37 @@ oparg = CURRENT_OPARG(); mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(mod_or_class_dict, name, &v); + LOAD_SP(); + if (err < 0) { JUMP_TO_ERROR(); } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + SAVE_SP(); + err = PyDict_GetItemRef(GLOBALS(), name, &v); + LOAD_SP(); + if (err < 0) { JUMP_TO_ERROR(); } if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &v); + LOAD_SP(); + if (err < 0) { JUMP_TO_ERROR(); } if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); JUMP_TO_ERROR(); } } } - Py_DECREF(mod_or_class_dict); + INTERPRETER_DECREF(mod_or_class_dict); stack_pointer[-1] = v; break; } @@ -1467,15 +1533,19 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { + SAVE_SP(); res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); + LOAD_SP(); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); } if (true) JUMP_TO_ERROR(); } @@ -1483,14 +1553,22 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR(); + SAVE_SP(); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &res); + LOAD_SP(); + if (err < 0) JUMP_TO_ERROR(); if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR(); + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &res); + LOAD_SP(); + if (err < 0) JUMP_TO_ERROR(); if (res == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); if (true) JUMP_TO_ERROR(); } } @@ -1578,10 +1656,12 @@ oparg = CURRENT_OPARG(); PyObject *v = GETLOCAL(oparg); if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + LOAD_SP(); if (1) JUMP_TO_ERROR(); } SETLOCAL(oparg, NULL); @@ -1608,10 +1688,12 @@ // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); JUMP_TO_ERROR(); } - Py_DECREF(oldobj); + INTERPRETER_DECREF(oldobj); break; } @@ -1624,18 +1706,23 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(class_dict, name, &value); + LOAD_SP(); + if (err < 0) { JUMP_TO_ERROR(); } if (!value) { PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); JUMP_TO_ERROR(); } } - Py_DECREF(class_dict); + INTERPRETER_DECREF(class_dict); stack_pointer[-1] = value; break; } @@ -1646,7 +1733,9 @@ PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); if (true) JUMP_TO_ERROR(); } stack_pointer[0] = value; @@ -1659,7 +1748,9 @@ oparg = CURRENT_OPARG(); v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + SAVE_SP(); PyCell_SetTakeRef(cell, v); + LOAD_SP(); stack_pointer += -1; break; } @@ -1686,7 +1777,7 @@ pieces = &stack_pointer[-oparg]; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + INTERPRETER_DECREF(pieces[_i]); } if (str == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg] = str; @@ -1724,8 +1815,11 @@ oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + LOAD_SP(); if (none_val == NULL) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { @@ -1734,11 +1828,12 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); + LOAD_SP(); if (true) JUMP_TO_ERROR(); } assert(Py_IsNone(none_val)); - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); stack_pointer += -1; break; } @@ -1749,8 +1844,10 @@ oparg = CURRENT_OPARG(); iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); + LOAD_SP(); + INTERPRETER_DECREF(iterable); if (err < 0) JUMP_TO_ERROR(); stack_pointer += -1; break; @@ -1763,12 +1860,14 @@ PyObject *map; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg*2]; + SAVE_SP(); map = _PyDict_FromItems( values, 2, values+1, 2, oparg); + LOAD_SP(); for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + INTERPRETER_DECREF(values[_i]); } if (map == NULL) JUMP_TO_ERROR(); stack_pointer[-oparg*2] = map; @@ -1780,22 +1879,27 @@ int err; PyObject *ann_dict; if (LOCALS() == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); + LOAD_SP(); if (true) JUMP_TO_ERROR(); } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR(); + SAVE_SP(); + err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + LOAD_SP(); + if (err < 0) JUMP_TO_ERROR(); if (ann_dict == NULL) { ann_dict = PyDict_New(); if (ann_dict == NULL) JUMP_TO_ERROR(); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); - Py_DECREF(ann_dict); + INTERPRETER_DECREF(ann_dict); if (err) JUMP_TO_ERROR(); } else { - Py_DECREF(ann_dict); + INTERPRETER_DECREF(ann_dict); } break; } @@ -1809,13 +1913,15 @@ values = &stack_pointer[-1 - oparg]; assert(PyTuple_CheckExact(keys)); assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); + SAVE_SP(); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); + LOAD_SP(); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + INTERPRETER_DECREF(values[_i]); } - Py_DECREF(keys); + INTERPRETER_DECREF(keys); if (map == NULL) JUMP_TO_ERROR(); stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; @@ -1832,15 +1938,17 @@ int err = PyDict_Update(dict, update); LOAD_SP(); if (err < 0) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - Py_DECREF(update); + LOAD_SP(); + INTERPRETER_DECREF(update); if (true) JUMP_TO_ERROR(); } - Py_DECREF(update); + INTERPRETER_DECREF(update); stack_pointer += -1; break; } @@ -1857,11 +1965,13 @@ int err = _PyDict_MergeEx(dict, update, 2); LOAD_SP(); if (err < 0) { + SAVE_SP(); _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); + LOAD_SP(); + INTERPRETER_DECREF(update); if (true) JUMP_TO_ERROR(); } - Py_DECREF(update); + INTERPRETER_DECREF(update); stack_pointer += -1; break; } @@ -1877,7 +1987,10 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) JUMP_TO_ERROR(); + SAVE_SP(); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, key, value); + LOAD_SP(); + if (err != 0) JUMP_TO_ERROR(); stack_pointer += -2; break; } @@ -1904,10 +2017,12 @@ } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + SAVE_SP(); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + LOAD_SP(); + INTERPRETER_DECREF(global_super); + INTERPRETER_DECREF(class); + INTERPRETER_DECREF(self); if (attr == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = attr; stack_pointer += -2; @@ -1937,18 +2052,20 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; + SAVE_SP(); attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); + LOAD_SP(); + INTERPRETER_DECREF(global_super); + INTERPRETER_DECREF(class); if (attr == NULL) { - Py_DECREF(self); + INTERPRETER_DECREF(self); if (true) JUMP_TO_ERROR(); } if (method_found) { self_or_null = self; // transfer ownership } else { - Py_DECREF(self); + INTERPRETER_DECREF(self); self_or_null = NULL; } stack_pointer[-3] = attr; @@ -1985,7 +2102,7 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ - Py_DECREF(owner); + INTERPRETER_DECREF(owner); if (attr == NULL) JUMP_TO_ERROR(); self_or_null = NULL; } @@ -1995,7 +2112,7 @@ SAVE_SP(); attr = PyObject_GetAttr(owner, name); LOAD_SP(); - Py_DECREF(owner); + INTERPRETER_DECREF(owner); if (attr == NULL) JUMP_TO_ERROR(); } stack_pointer[-1] = attr; @@ -2044,7 +2161,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; break; } @@ -2064,7 +2181,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; @@ -2109,7 +2226,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -2165,7 +2282,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; stack_pointer += (oparg & 1); @@ -2188,7 +2305,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; break; } @@ -2209,7 +2326,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; @@ -2245,7 +2362,7 @@ assert(descr != NULL); attr = Py_NewRef(descr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; break; } @@ -2261,7 +2378,7 @@ assert(descr != NULL); attr = Py_NewRef(descr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; stack_pointer += 1; @@ -2305,9 +2422,9 @@ _PyDictValues_AddToInsertionOrder(values, index); } else { - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); } - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer += -2; break; } @@ -2361,7 +2478,7 @@ new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); STAT_INC(STORE_ATTR, hit); /* Ensure dict is GC tracked if it needs to be */ if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { @@ -2369,7 +2486,7 @@ } /* PEP 509 */ dict->ma_version_tag = new_version; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); stack_pointer += -2; break; } @@ -2384,8 +2501,10 @@ STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); + if (old_value != NULL) { + INTERPRETER_DECREF(old_value); + } + INTERPRETER_DECREF(owner); stack_pointer += -2; break; } @@ -2401,12 +2520,12 @@ SAVE_SP(); res = PyObject_RichCompare(left, right, oparg >> 5); LOAD_SP(); - Py_DECREF(left); - Py_DECREF(right); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res == NULL) JUMP_TO_ERROR(); if (oparg & 16) { int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); + INTERPRETER_DECREF(res); if (res_bool < 0) JUMP_TO_ERROR(); res = res_bool ? Py_True : Py_False; } @@ -2497,8 +2616,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); b = res ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -2515,8 +2634,8 @@ SAVE_SP(); int res = PySequence_Contains(right, left); LOAD_SP(); - Py_DECREF(left); - Py_DECREF(right); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; @@ -2537,9 +2656,11 @@ } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! + SAVE_SP(); int res = _PySet_Contains((PySetObject *)right, left); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; @@ -2559,9 +2680,11 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CONTAINS_OP, hit); + SAVE_SP(); int res = PyDict_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) JUMP_TO_ERROR(); b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; @@ -2577,16 +2700,18 @@ match_type = stack_pointer[-1]; exc_value = stack_pointer[-2]; if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); + INTERPRETER_DECREF(exc_value); + INTERPRETER_DECREF(match_type); if (true) JUMP_TO_ERROR(); } match = NULL; rest = NULL; + SAVE_SP(); int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); + LOAD_SP(); + INTERPRETER_DECREF(exc_value); + INTERPRETER_DECREF(match_type); if (res < 0) JUMP_TO_ERROR(); assert((match == NULL) == (rest == NULL)); if (match == NULL) JUMP_TO_ERROR(); @@ -2605,12 +2730,15 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); + SAVE_SP(); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(right); if (true) JUMP_TO_ERROR(); } int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(right); b = res ? Py_True : Py_False; stack_pointer[-1] = b; break; @@ -2629,7 +2757,7 @@ } else { b = Py_False; - Py_DECREF(value); + INTERPRETER_DECREF(value); } stack_pointer[-1] = b; break; @@ -2661,10 +2789,12 @@ // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); + SAVE_SP(); attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); + LOAD_SP(); + INTERPRETER_DECREF(subject); + INTERPRETER_DECREF(type); + INTERPRETER_DECREF(names); if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -2707,7 +2837,9 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). + SAVE_SP(); values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + LOAD_SP(); if (values_or_none == NULL) JUMP_TO_ERROR(); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -2719,8 +2851,10 @@ PyObject *iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ + SAVE_SP(); iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); + LOAD_SP(); + INTERPRETER_DECREF(iterable); if (iter == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = iter; break; @@ -2748,11 +2882,13 @@ } else { /* `iterable` is not a generator. */ + SAVE_SP(); iter = PyObject_GetIter(iterable); + LOAD_SP(); if (iter == NULL) { JUMP_TO_ERROR(); } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); } stack_pointer[-1] = iter; break; @@ -2765,13 +2901,17 @@ PyObject *next; iter = stack_pointer[-1]; /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + SAVE_SP(); next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next == NULL) { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { JUMP_TO_ERROR(); } + SAVE_SP(); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ /* The translator sets the deopt target just past the matching END_FOR */ @@ -2973,7 +3113,7 @@ tb = Py_None; } else { - Py_DECREF(tb); + INTERPRETER_DECREF(tb); } assert(PyLong_Check(lasti)); (void)lasti; // Shut up compiler warning if asserts are off @@ -3080,7 +3220,7 @@ assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); + INTERPRETER_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; break; @@ -3096,7 +3236,7 @@ assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); + INTERPRETER_DECREF(owner); attr = Py_NewRef(descr); stack_pointer[-1] = attr; break; @@ -3162,10 +3302,12 @@ assert(Py_TYPE(callable) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. stack_pointer += -2 - oparg; @@ -3237,7 +3379,7 @@ method = ((PyMethodObject *)callable)->im_func; assert(PyFunction_Check(method)); Py_INCREF(method); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); stack_pointer[-2 - oparg] = method; stack_pointer[-1 - oparg] = self; break; @@ -3283,9 +3425,9 @@ NULL); LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; @@ -3321,7 +3463,7 @@ stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS func = Py_NewRef(((PyMethodObject *)callable)->im_func); stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + INTERPRETER_DECREF(callable); stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = self; break; @@ -3551,7 +3693,7 @@ SAVE_SP(); res = Py_NewRef(Py_TYPE(arg)); LOAD_SP(); - Py_DECREF(arg); + INTERPRETER_DECREF(arg); stack_pointer[-3] = res; stack_pointer += -2; break; @@ -3579,7 +3721,7 @@ SAVE_SP(); res = PyObject_Str(arg); LOAD_SP(); - Py_DECREF(arg); + INTERPRETER_DECREF(arg); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; @@ -3608,7 +3750,7 @@ SAVE_SP(); res = PySequence_Tuple(arg); LOAD_SP(); - Py_DECREF(arg); + INTERPRETER_DECREF(arg); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-3] = res; stack_pointer += -2; @@ -3660,9 +3802,9 @@ LOAD_SP(); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(tp); + INTERPRETER_DECREF(tp); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3710,8 +3852,8 @@ LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); + INTERPRETER_DECREF(arg); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3753,9 +3895,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3796,9 +3938,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3842,8 +3984,8 @@ if (res == NULL) { GOTO_ERROR(error); } - Py_DECREF(callable); - Py_DECREF(arg); + INTERPRETER_DECREF(callable); + INTERPRETER_DECREF(arg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3887,9 +4029,9 @@ if (res == NULL) { GOTO_ERROR(error); } - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); + INTERPRETER_DECREF(inst); + INTERPRETER_DECREF(cls); + INTERPRETER_DECREF(callable); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; break; @@ -3942,9 +4084,9 @@ LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); + INTERPRETER_DECREF(self); + INTERPRETER_DECREF(arg); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -3991,9 +4133,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -4047,8 +4189,8 @@ LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); + INTERPRETER_DECREF(self); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -4095,9 +4237,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -4116,9 +4258,11 @@ PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; + SAVE_SP(); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); + LOAD_SP(); + INTERPRETER_DECREF(codeobj); if (func_obj == NULL) { JUMP_TO_ERROR(); } @@ -4203,9 +4347,9 @@ stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); + INTERPRETER_DECREF(start); + INTERPRETER_DECREF(stop); + if (step != NULL) { INTERPRETER_DECREF(step); } if (slice == NULL) JUMP_TO_ERROR(); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -4223,7 +4367,7 @@ SAVE_SP(); result = conv_fn(value); LOAD_SP(); - Py_DECREF(value); + INTERPRETER_DECREF(value); if (result == NULL) JUMP_TO_ERROR(); stack_pointer[-1] = result; break; @@ -4236,8 +4380,10 @@ /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { + SAVE_SP(); res = PyObject_Format(value, NULL); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) JUMP_TO_ERROR(); } else { @@ -4253,9 +4399,11 @@ PyObject *res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; + SAVE_SP(); res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); + LOAD_SP(); + INTERPRETER_DECREF(value); + INTERPRETER_DECREF(fmt_spec); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; @@ -4285,8 +4433,8 @@ SAVE_SP(); res = _PyEval_BinaryOps[oparg](lhs, rhs); LOAD_SP(); - Py_DECREF(lhs); - Py_DECREF(rhs); + INTERPRETER_DECREF(lhs); + INTERPRETER_DECREF(rhs); if (res == NULL) JUMP_TO_ERROR(); stack_pointer[-2] = res; stack_pointer += -1; @@ -4348,7 +4496,7 @@ val = stack_pointer[-1]; stack_pointer += -1; if (!Py_IsNone(val)) { - Py_DECREF(val); + INTERPRETER_DECREF(val); if (1) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4365,7 +4513,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - Py_DECREF(val); + INTERPRETER_DECREF(val); break; } @@ -4443,7 +4591,7 @@ PyObject *value; pop = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND(); - Py_DECREF(pop); + INTERPRETER_DECREF(pop); value = ptr; stack_pointer[-1] = value; break; @@ -4513,7 +4661,7 @@ if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); if (optimized < 0) { - Py_DECREF(previous); + INTERPRETER_DECREF(previous); tstate->previous_executor = Py_None; GOTO_UNWIND(); } @@ -4548,7 +4696,7 @@ if (optimized <= 0) { exit->temperature = restart_backoff_counter(exit->temperature); if (optimized < 0) { - Py_DECREF(current_executor); + INTERPRETER_DECREF(current_executor); tstate->previous_executor = Py_None; GOTO_UNWIND(); } @@ -4564,7 +4712,7 @@ case _START_EXECUTOR: { PyObject *executor = (PyObject *)CURRENT_OPERAND(); - Py_DECREF(tstate->previous_executor); + INTERPRETER_DECREF(tstate->previous_executor); tstate->previous_executor = NULL; #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8a1d4080f5edd0..ab3d21a5989f5f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -38,16 +38,16 @@ "(missed __aexit__ method)", Py_TYPE(mgr)->tp_name); } - Py_DECREF(enter); + INTERPRETER_DECREF(enter); goto error; } - Py_DECREF(mgr); + INTERPRETER_DECREF(mgr); SAVE_SP(); res = PyObject_CallNoArgs(enter); LOAD_SP(); - Py_DECREF(enter); + INTERPRETER_DECREF(enter); if (res == NULL) { - Py_DECREF(exit); + INTERPRETER_DECREF(exit); if (true) goto pop_1_error; } stack_pointer[-1] = exit; @@ -90,16 +90,16 @@ "(missed __exit__ method)", Py_TYPE(mgr)->tp_name); } - Py_DECREF(enter); + INTERPRETER_DECREF(enter); goto error; } - Py_DECREF(mgr); + INTERPRETER_DECREF(mgr); SAVE_SP(); res = PyObject_CallNoArgs(enter); LOAD_SP(); - Py_DECREF(enter); + INTERPRETER_DECREF(enter); if (res == NULL) { - Py_DECREF(exit); + INTERPRETER_DECREF(exit); if (true) goto pop_1_error; } stack_pointer[-1] = exit; @@ -127,7 +127,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); @@ -142,8 +144,8 @@ SAVE_SP(); res = _PyEval_BinaryOps[oparg](lhs, rhs); LOAD_SP(); - Py_DECREF(lhs); - Py_DECREF(rhs); + INTERPRETER_DECREF(lhs); + INTERPRETER_DECREF(rhs); if (res == NULL) goto pop_2_error; } stack_pointer[-2] = res; @@ -272,7 +274,9 @@ */ assert(Py_REFCNT(left) >= 2); _Py_DECREF_NO_DEALLOC(left); + SAVE_SP(); PyUnicode_Append(target_local, right); + LOAD_SP(); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. @@ -419,10 +423,12 @@ res = NULL; } else { + SAVE_SP(); res = PyObject_GetItem(container, slice); - Py_DECREF(slice); + LOAD_SP(); + INTERPRETER_DECREF(slice); } - Py_DECREF(container); + INTERPRETER_DECREF(container); if (res == NULL) goto pop_3_error; stack_pointer[-3] = res; stack_pointer += -2; @@ -448,7 +454,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_BinarySubscr(container, sub, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); @@ -460,8 +468,8 @@ SAVE_SP(); res = PyObject_GetItem(container, sub); LOAD_SP(); - Py_DECREF(container); - Py_DECREF(sub); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (res == NULL) goto pop_2_error; } stack_pointer[-2] = res; @@ -482,12 +490,14 @@ dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); + SAVE_SP(); int rc = PyDict_GetItemRef(dict, sub, &res); + LOAD_SP(); if (rc == 0) { _PyErr_SetKeyError(sub); } - Py_DECREF(dict); - Py_DECREF(sub); + INTERPRETER_DECREF(dict); + INTERPRETER_DECREF(sub); if (rc <= 0) goto pop_2_error; // not found or error stack_pointer[-2] = res; @@ -550,7 +560,7 @@ assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + INTERPRETER_DECREF(list); stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -578,7 +588,7 @@ STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); + INTERPRETER_DECREF(str); stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -606,7 +616,7 @@ assert(res != NULL); Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); + INTERPRETER_DECREF(tuple); stack_pointer[-2] = res; stack_pointer += -1; DISPATCH(); @@ -623,13 +633,15 @@ values = &stack_pointer[-1 - oparg]; assert(PyTuple_CheckExact(keys)); assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); + SAVE_SP(); map = _PyDict_FromItems( &PyTuple_GET_ITEM(keys, 0), 1, values, 1, oparg); + LOAD_SP(); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); + INTERPRETER_DECREF(values[_i]); } - Py_DECREF(keys); + INTERPRETER_DECREF(keys); if (map == NULL) { stack_pointer += -1 - oparg; goto error; } stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; @@ -657,12 +669,14 @@ PyObject **values; PyObject *map; values = &stack_pointer[-oparg*2]; + SAVE_SP(); map = _PyDict_FromItems( values, 2, values+1, 2, oparg); + LOAD_SP(); for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + INTERPRETER_DECREF(values[_i]); } if (map == NULL) { stack_pointer += -oparg*2; goto error; } stack_pointer[-oparg*2] = map; @@ -683,12 +697,15 @@ int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); + if (err == 0) { + SAVE_SP(); + err = PySet_Add(set, item); + LOAD_SP(); + } + INTERPRETER_DECREF(item); } if (err != 0) { - Py_DECREF(set); + INTERPRETER_DECREF(set); if (true) { stack_pointer += -oparg; goto error; } } stack_pointer[-oparg] = set; @@ -708,9 +725,9 @@ stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); + INTERPRETER_DECREF(start); + INTERPRETER_DECREF(stop); + if (step != NULL) { INTERPRETER_DECREF(step); } if (slice == NULL) { stack_pointer += -2 - ((oparg == 3) ? 1 : 0); goto error; } stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -726,7 +743,7 @@ pieces = &stack_pointer[-oparg]; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + INTERPRETER_DECREF(pieces[_i]); } if (str == NULL) { stack_pointer += -oparg; goto error; } stack_pointer[-oparg] = str; @@ -778,7 +795,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); @@ -801,7 +820,7 @@ args[0] = Py_NewRef(self); PyObject *method = ((PyMethodObject *)callable)->im_func; args[-1] = Py_NewRef(method); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); callable = method; } // Check if the call can be inlined or not @@ -811,10 +830,12 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); // The frame has stolen all the arguments from the stack, @@ -835,6 +856,7 @@ if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; + SAVE_SP(); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -848,11 +870,12 @@ Py_CLEAR(res); } } + LOAD_SP(); } assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } @@ -899,7 +922,7 @@ if (self == NULL) { goto error; } - Py_DECREF(tp); + INTERPRETER_DECREF(tp); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1); assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK); @@ -959,7 +982,7 @@ stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _INIT_CALL_PY_EXACT_ARGS func = Py_NewRef(((PyMethodObject *)callable)->im_func); stack_pointer[-2 - oparg] = func; // This is used by CALL, upon deoptimization - Py_DECREF(callable); + INTERPRETER_DECREF(callable); } // _CHECK_FUNCTION_VERSION callable = func; @@ -1063,7 +1086,7 @@ method = ((PyMethodObject *)callable)->im_func; assert(PyFunction_Check(method)); Py_INCREF(method); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); } // _PY_FRAME_GENERAL args = &stack_pointer[-oparg]; @@ -1079,10 +1102,12 @@ assert(Py_TYPE(callable) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. stack_pointer += -2 - oparg; @@ -1146,9 +1171,9 @@ LOAD_SP(); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(tp); + INTERPRETER_DECREF(tp); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1196,9 +1221,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1245,9 +1270,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1295,8 +1320,8 @@ LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); + INTERPRETER_DECREF(arg); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1329,32 +1354,43 @@ if (check_args_iterable(tstate, func, callargs) < 0) { goto error; } + SAVE_SP(); PyObject *tuple = PySequence_Tuple(callargs); + LOAD_SP(); if (tuple == NULL) { goto error; } - Py_SETREF(callargs, tuple); + INTERPRETER_DECREF(callargs); + callargs = tuple; } assert(PyTuple_CheckExact(callargs)); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); + LOAD_SP(); if (err) goto error; + SAVE_SP(); result = PyObject_Call(func, callargs, kwargs); + LOAD_SP(); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result == NULL) { + SAVE_SP(); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); + LOAD_SP(); } else { + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); + LOAD_SP(); if (err < 0) { Py_CLEAR(result); } @@ -1369,9 +1405,11 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, (PyFunctionObject *)func, locals, nargs, callargs, kwargs); + LOAD_SP(); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -1385,9 +1423,9 @@ result = PyObject_Call(func, callargs, kwargs); LOAD_SP(); } - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); + INTERPRETER_DECREF(func); + INTERPRETER_DECREF(callargs); + if (kwargs != NULL) { INTERPRETER_DECREF(kwargs); } assert(PEEK(2 + (oparg & 1)) == NULL); if (result == NULL) { stack_pointer += -3 - (oparg & 1); goto error; } stack_pointer[-3 - (oparg & 1)] = result; @@ -1407,7 +1445,7 @@ SAVE_SP(); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); LOAD_SP(); - Py_DECREF(value); + INTERPRETER_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); @@ -1426,8 +1464,8 @@ SAVE_SP(); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); LOAD_SP(); - Py_DECREF(value2); - Py_DECREF(value1); + INTERPRETER_DECREF(value2); + INTERPRETER_DECREF(value1); if (res == NULL) goto pop_2_error; stack_pointer[-2] = res; stack_pointer += -1; @@ -1471,9 +1509,9 @@ if (res == NULL) { GOTO_ERROR(error); } - Py_DECREF(inst); - Py_DECREF(cls); - Py_DECREF(callable); + INTERPRETER_DECREF(inst); + INTERPRETER_DECREF(cls); + INTERPRETER_DECREF(callable); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; DISPATCH(); @@ -1508,7 +1546,7 @@ args[0] = Py_NewRef(self); PyObject *method = ((PyMethodObject *)callable)->im_func; args[-1] = Py_NewRef(method); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); callable = method; } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames); @@ -1519,11 +1557,13 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, positional_args, kwnames ); - Py_DECREF(kwnames); + LOAD_SP(); + INTERPRETER_DECREF(kwnames); // Manipulate stack directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 3); // The frame has stolen all the arguments from the stack, @@ -1545,6 +1585,7 @@ if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : args[0]; + SAVE_SP(); if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -1558,12 +1599,13 @@ Py_CLEAR(res); } } + LOAD_SP(); } - Py_DECREF(kwnames); + INTERPRETER_DECREF(kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } if (res == NULL) { stack_pointer += -3 - oparg; goto error; } stack_pointer[-3 - oparg] = res; @@ -1608,8 +1650,8 @@ if (res == NULL) { GOTO_ERROR(error); } - Py_DECREF(callable); - Py_DECREF(arg); + INTERPRETER_DECREF(callable); + INTERPRETER_DECREF(arg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; DISPATCH(); @@ -1637,8 +1679,8 @@ if (_PyList_AppendTakeRef((PyListObject *)self, arg) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } - Py_DECREF(self); - Py_DECREF(callable); + INTERPRETER_DECREF(self); + INTERPRETER_DECREF(callable); STACK_SHRINK(3); // Skip POP_TOP assert(next_instr->op.code == POP_TOP); @@ -1684,9 +1726,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1736,9 +1778,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } - Py_DECREF(callable); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1789,8 +1831,8 @@ LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); + INTERPRETER_DECREF(self); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1841,9 +1883,9 @@ LOAD_SP(); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); + INTERPRETER_DECREF(self); + INTERPRETER_DECREF(arg); + INTERPRETER_DECREF(callable); if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } // _CHECK_PERIODIC @@ -1892,9 +1934,9 @@ NULL); LOAD_SP(); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); + INTERPRETER_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + INTERPRETER_DECREF(args[i]); } if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } @@ -2020,10 +2062,12 @@ assert(Py_TYPE(callable) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + SAVE_SP(); new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)callable, locals, args, total_args, NULL ); + LOAD_SP(); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. stack_pointer += -2 - oparg; @@ -2080,7 +2124,7 @@ SAVE_SP(); res = PyObject_Str(arg); LOAD_SP(); - Py_DECREF(arg); + INTERPRETER_DECREF(arg); if (res == NULL) goto pop_3_error; } // _CHECK_PERIODIC @@ -2115,7 +2159,7 @@ SAVE_SP(); res = PySequence_Tuple(arg); LOAD_SP(); - Py_DECREF(arg); + INTERPRETER_DECREF(arg); if (res == NULL) goto pop_3_error; } // _CHECK_PERIODIC @@ -2148,7 +2192,7 @@ SAVE_SP(); res = Py_NewRef(Py_TYPE(arg)); LOAD_SP(); - Py_DECREF(arg); + INTERPRETER_DECREF(arg); stack_pointer[-3] = res; stack_pointer += -2; DISPATCH(); @@ -2165,16 +2209,18 @@ match_type = stack_pointer[-1]; exc_value = stack_pointer[-2]; if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); + INTERPRETER_DECREF(exc_value); + INTERPRETER_DECREF(match_type); if (true) goto pop_2_error; } match = NULL; rest = NULL; + SAVE_SP(); int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); + LOAD_SP(); + INTERPRETER_DECREF(exc_value); + INTERPRETER_DECREF(match_type); if (res < 0) goto pop_2_error; assert((match == NULL) == (rest == NULL)); if (match == NULL) goto pop_2_error; @@ -2196,12 +2242,15 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); + SAVE_SP(); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(right); if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(right); b = res ? Py_True : Py_False; stack_pointer[-1] = b; DISPATCH(); @@ -2224,14 +2273,16 @@ assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { value = Py_NewRef(((PyStopIterationObject *)exc_value)->value); - Py_DECREF(sub_iter); - Py_DECREF(last_sent_val); - Py_DECREF(exc_value); + INTERPRETER_DECREF(sub_iter); + INTERPRETER_DECREF(last_sent_val); + INTERPRETER_DECREF(exc_value); none = Py_None; } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } stack_pointer[-3] = none; @@ -2259,7 +2310,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_CompareOp(left, right, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); @@ -2272,12 +2325,12 @@ SAVE_SP(); res = PyObject_RichCompare(left, right, oparg >> 5); LOAD_SP(); - Py_DECREF(left); - Py_DECREF(right); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res == NULL) goto pop_2_error; if (oparg & 16) { int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); + INTERPRETER_DECREF(res); if (res_bool < 0) goto pop_2_error; res = res_bool ? Py_True : Py_False; } @@ -2410,7 +2463,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ContainsOp(right, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(CONTAINS_OP, deferred); @@ -2422,8 +2477,8 @@ SAVE_SP(); int res = PySequence_Contains(right, left); LOAD_SP(); - Py_DECREF(left); - Py_DECREF(right); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; } @@ -2445,9 +2500,11 @@ left = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(right), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); + SAVE_SP(); int res = PyDict_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; @@ -2469,9 +2526,11 @@ DEOPT_IF(!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right)), CONTAINS_OP); STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! + SAVE_SP(); int res = _PySet_Contains((PySetObject *)right, left); - Py_DECREF(left); - Py_DECREF(right); + LOAD_SP(); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; stack_pointer[-2] = b; @@ -2492,7 +2551,7 @@ SAVE_SP(); result = conv_fn(value); LOAD_SP(); - Py_DECREF(value); + INTERPRETER_DECREF(value); if (result == NULL) goto pop_1_error; stack_pointer[-1] = result; DISPATCH(); @@ -2536,8 +2595,10 @@ PyObject *owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyObject_DelAttr(owner, name); - Py_DECREF(owner); + LOAD_SP(); + INTERPRETER_DECREF(owner); if (err) goto pop_1_error; stack_pointer += -1; DISPATCH(); @@ -2552,10 +2613,12 @@ // Fortunately we don't need its superpower. PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); goto error; } - Py_DECREF(oldobj); + INTERPRETER_DECREF(oldobj); DISPATCH(); } @@ -2565,10 +2628,12 @@ INSTRUCTION_STATS(DELETE_FAST); PyObject *v = GETLOCAL(oparg); if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); + LOAD_SP(); if (1) goto error; } SETLOCAL(oparg, NULL); @@ -2580,14 +2645,18 @@ next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_Pop(GLOBALS(), name, NULL); + LOAD_SP(); // Can't use ERROR_IF here. if (err < 0) { goto error; } if (err == 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); goto error; } DISPATCH(); @@ -2601,16 +2670,22 @@ PyObject *ns = LOCALS(); int err; if (ns == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); + LOAD_SP(); goto error; } + SAVE_SP(); err = PyObject_DelItem(ns, name); + LOAD_SP(); // Can't use ERROR_IF here. if (err != 0) { + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); goto error; } DISPATCH(); @@ -2628,8 +2703,8 @@ SAVE_SP(); int err = PyObject_DelItem(container, sub); LOAD_SP(); - Py_DECREF(container); - Py_DECREF(sub); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (err) goto pop_2_error; stack_pointer += -2; DISPATCH(); @@ -2649,11 +2724,13 @@ int err = _PyDict_MergeEx(dict, update, 2); LOAD_SP(); if (err < 0) { + SAVE_SP(); _PyEval_FormatKwargsError(tstate, callable, update); - Py_DECREF(update); + LOAD_SP(); + INTERPRETER_DECREF(update); if (true) goto pop_1_error; } - Py_DECREF(update); + INTERPRETER_DECREF(update); stack_pointer += -1; DISPATCH(); } @@ -2670,15 +2747,17 @@ int err = PyDict_Update(dict, update); LOAD_SP(); if (err < 0) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(update)->tp_name); } - Py_DECREF(update); + LOAD_SP(); + INTERPRETER_DECREF(update); if (true) goto pop_1_error; } - Py_DECREF(update); + INTERPRETER_DECREF(update); stack_pointer += -1; DISPATCH(); } @@ -2694,13 +2773,15 @@ awaitable = stack_pointer[-2]; assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - Py_DECREF(awaitable); - Py_DECREF(exc); + INTERPRETER_DECREF(awaitable); + INTERPRETER_DECREF(exc); } else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } stack_pointer += -2; @@ -2713,7 +2794,7 @@ INSTRUCTION_STATS(END_FOR); PyObject *value; value = stack_pointer[-1]; - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer += -1; DISPATCH(); } @@ -2726,7 +2807,7 @@ PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; - Py_DECREF(receiver); + INTERPRETER_DECREF(receiver); stack_pointer[-2] = value; stack_pointer += -1; DISPATCH(); @@ -2803,8 +2884,10 @@ /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { + SAVE_SP(); res = PyObject_Format(value, NULL); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) goto pop_1_error; } else { @@ -2823,9 +2906,11 @@ PyObject *res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; + SAVE_SP(); res = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_DECREF(fmt_spec); + LOAD_SP(); + INTERPRETER_DECREF(value); + INTERPRETER_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; stack_pointer[-2] = res; stack_pointer += -1; @@ -2849,7 +2934,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ForIter(iter, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); @@ -2867,13 +2954,15 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } + SAVE_SP(); monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); + INTERPRETER_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instruction */ JUMPBY(oparg + 2); @@ -2956,10 +3045,10 @@ #ifndef Py_GIL_DISABLED if (seq != NULL) { it->it_seq = NULL; - Py_DECREF(seq); + INTERPRETER_DECREF(seq); } #endif - Py_DECREF(iter); + INTERPRETER_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ JUMPBY(oparg + 2); @@ -3001,7 +3090,7 @@ STAT_INC(FOR_ITER, hit); if (r->len <= 0) { STACK_SHRINK(1); - Py_DECREF(r); + INTERPRETER_DECREF(r); // Jump over END_FOR and POP_TOP instructions. JUMPBY(oparg + 2); DISPATCH(); @@ -3045,9 +3134,9 @@ if (seq == NULL || it->it_index >= PyTuple_GET_SIZE(seq)) { if (seq != NULL) { it->it_seq = NULL; - Py_DECREF(seq); + INTERPRETER_DECREF(seq); } - Py_DECREF(iter); + INTERPRETER_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR and POP_TOP instructions */ JUMPBY(oparg + 2); @@ -3081,15 +3170,19 @@ getter = type->tp_as_async->am_aiter; } if (getter == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - Py_DECREF(obj); + LOAD_SP(); + INTERPRETER_DECREF(obj); if (true) goto pop_1_error; } + SAVE_SP(); iter = (*getter)(obj); - Py_DECREF(obj); + LOAD_SP(); + INTERPRETER_DECREF(obj); if (iter == NULL) goto pop_1_error; if (Py_TYPE(iter)->tp_as_async == NULL || Py_TYPE(iter)->tp_as_async->am_anext == NULL) { @@ -3097,7 +3190,7 @@ "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter)->tp_name); - Py_DECREF(iter); + INTERPRETER_DECREF(iter); if (true) goto pop_1_error; } stack_pointer[-1] = iter; @@ -3115,7 +3208,9 @@ PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); if (PyAsyncGen_CheckExact(aiter)) { + SAVE_SP(); awaitable = type->tp_as_async->am_anext(aiter); + LOAD_SP(); if (awaitable == NULL) { goto error; } @@ -3124,29 +3219,37 @@ getter = type->tp_as_async->am_anext; } if (getter != NULL) { + SAVE_SP(); next_iter = (*getter)(aiter); + LOAD_SP(); if (next_iter == NULL) { goto error; } } else { + SAVE_SP(); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an iterator with " "__anext__ method, got %.100s", type->tp_name); + LOAD_SP(); goto error; } + SAVE_SP(); awaitable = _PyCoro_GetAwaitableIter(next_iter); + LOAD_SP(); if (awaitable == NULL) { + SAVE_SP(); _PyErr_FormatFromCause( PyExc_TypeError, "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); + LOAD_SP(); + INTERPRETER_DECREF(next_iter); goto error; } else { - Py_DECREF(next_iter); + INTERPRETER_DECREF(next_iter); } } stack_pointer[0] = awaitable; @@ -3161,18 +3264,22 @@ PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; + SAVE_SP(); iter = _PyCoro_GetAwaitableIter(iterable); + LOAD_SP(); if (iter == NULL) { + SAVE_SP(); _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + LOAD_SP(); } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { /* `iter` is a coroutine object that is being awaited, `yf` is a pointer to the current awaitable being awaited on. */ - Py_DECREF(yf); + INTERPRETER_DECREF(yf); Py_CLEAR(iter); _PyErr_SetString(tstate, PyExc_RuntimeError, "coroutine is being awaited already"); @@ -3192,8 +3299,10 @@ PyObject *iter; iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ + SAVE_SP(); iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); + LOAD_SP(); + INTERPRETER_DECREF(iterable); if (iter == NULL) goto pop_1_error; stack_pointer[-1] = iter; DISPATCH(); @@ -3241,11 +3350,13 @@ } else { /* `iterable` is not a generator. */ + SAVE_SP(); iter = PyObject_GetIter(iterable); + LOAD_SP(); if (iter == NULL) { goto error; } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); } stack_pointer[-1] = iter; DISPATCH(); @@ -3281,8 +3392,8 @@ SAVE_SP(); res = import_name(tstate, frame, name, fromlist, level); LOAD_SP(); - Py_DECREF(level); - Py_DECREF(fromlist); + INTERPRETER_DECREF(level); + INTERPRETER_DECREF(fromlist); if (res == NULL) goto pop_2_error; stack_pointer[-2] = res; stack_pointer += -1; @@ -3300,9 +3411,11 @@ PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args); + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + LOAD_SP(); if (err) goto error; PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); GO_TO_INSTRUCTION(CALL); @@ -3325,9 +3438,11 @@ PyObject *function = PEEK(oparg + 3); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args + 1); + SAVE_SP(); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); + LOAD_SP(); if (err) goto error; GO_TO_INSTRUCTION(CALL_KW); } @@ -3351,7 +3466,7 @@ goto error; } } - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer += -1; DISPATCH(); } @@ -3373,7 +3488,7 @@ goto error; } } - Py_DECREF(receiver); + INTERPRETER_DECREF(receiver); stack_pointer[-2] = value; stack_pointer += -1; DISPATCH(); @@ -3387,7 +3502,9 @@ /* Skip 1 cache entry */ _Py_CODEUNIT *target; PyObject *iter = TOP(); + SAVE_SP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); + LOAD_SP(); if (next != NULL) { PUSH(next); target = next_instr; @@ -3397,14 +3514,16 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } + SAVE_SP(); monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); + LOAD_SP(); } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); - Py_DECREF(iter); + INTERPRETER_DECREF(iter); /* Skip END_FOR and POP_TOP */ target = next_instr + oparg + 2; } @@ -3493,7 +3612,7 @@ offset = oparg; } else { - Py_DECREF(value); + INTERPRETER_DECREF(value); offset = 0; } #if ENABLE_SPECIALIZATION @@ -3516,7 +3635,7 @@ offset = 0; } else { - Py_DECREF(value); + INTERPRETER_DECREF(value); offset = oparg; } #if ENABLE_SPECIALIZATION @@ -3580,9 +3699,11 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); + SAVE_SP(); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); + LOAD_SP(); if (err) goto error; Py_INCREF(retval); assert(EMPTY()); @@ -3605,9 +3726,11 @@ INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; + SAVE_SP(); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, retval); + LOAD_SP(); if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); @@ -3680,8 +3803,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; - Py_DECREF(left); - Py_DECREF(right); + INTERPRETER_DECREF(left); + INTERPRETER_DECREF(right); b = res ? Py_True : Py_False; stack_pointer[-2] = b; stack_pointer += -1; @@ -3769,8 +3892,11 @@ PyObject *list; iterable = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + LOAD_SP(); if (none_val == NULL) { + SAVE_SP(); if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { @@ -3779,11 +3905,12 @@ "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); } - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); + LOAD_SP(); if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); - Py_DECREF(iterable); + INTERPRETER_DECREF(iterable); stack_pointer += -1; DISPATCH(); } @@ -3807,7 +3934,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadAttr(owner, next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); @@ -3839,7 +3968,7 @@ CALL that it's not a method call. meth | NULL | arg1 | ... | argN */ - Py_DECREF(owner); + INTERPRETER_DECREF(owner); if (attr == NULL) goto pop_1_error; self_or_null = NULL; } @@ -3849,7 +3978,7 @@ SAVE_SP(); attr = PyObject_GetAttr(owner, name); LOAD_SP(); - Py_DECREF(owner); + INTERPRETER_DECREF(owner); if (attr == NULL) goto pop_1_error; } } @@ -3884,7 +4013,7 @@ assert(descr != NULL); attr = Py_NewRef(descr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = null; @@ -3958,7 +4087,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4120,7 +4249,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4153,7 +4282,7 @@ assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); + INTERPRETER_DECREF(owner); attr = Py_NewRef(descr); } stack_pointer[-1] = attr; @@ -4194,7 +4323,7 @@ assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(owner); + INTERPRETER_DECREF(owner); attr = Py_NewRef(descr); } stack_pointer[-1] = attr; @@ -4260,7 +4389,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4313,7 +4442,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } /* Skip 5 cache entries */ stack_pointer[-1] = attr; @@ -4327,7 +4456,10 @@ next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); PyObject *bc; - if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; + SAVE_SP(); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc); + LOAD_SP(); + if (err < 0) goto error; if (bc == NULL) { _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); @@ -4379,7 +4511,9 @@ PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); if (true) goto error; } stack_pointer[0] = value; @@ -4461,18 +4595,23 @@ assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(class_dict, name, &value); + LOAD_SP(); + if (err < 0) { goto error; } if (!value) { PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); value = PyCell_GetRef(cell); if (value == NULL) { + SAVE_SP(); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + LOAD_SP(); goto error; } } - Py_DECREF(class_dict); + INTERPRETER_DECREF(class_dict); stack_pointer[-1] = value; DISPATCH(); } @@ -4485,26 +4624,37 @@ PyObject *v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(mod_or_class_dict, name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + SAVE_SP(); + err = PyDict_GetItemRef(GLOBALS(), name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); goto error; } } } - Py_DECREF(mod_or_class_dict); + INTERPRETER_DECREF(mod_or_class_dict); stack_pointer[-1] = v; DISPATCH(); } @@ -4526,7 +4676,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); @@ -4542,15 +4694,19 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { + SAVE_SP(); res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); + LOAD_SP(); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ + SAVE_SP(); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); } if (true) goto error; } @@ -4558,14 +4714,22 @@ else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + SAVE_SP(); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &res); + LOAD_SP(); + if (err < 0) goto error; if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &res); + LOAD_SP(); + if (err < 0) goto error; if (res == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); if (true) goto error; } } @@ -4660,8 +4824,10 @@ PyObject *locals; locals = LOCALS(); if (locals == NULL) { + SAVE_SP(); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + LOAD_SP(); if (true) goto error; } Py_INCREF(locals); @@ -4677,26 +4843,39 @@ PyObject *v; PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { + SAVE_SP(); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); + LOAD_SP(); if (true) goto error; } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + SAVE_SP(); + int err = PyMapping_GetOptionalItem(mod_or_class_dict, name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + SAVE_SP(); + err = PyDict_GetItemRef(GLOBALS(), name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + SAVE_SP(); + err = PyMapping_GetOptionalItem(BUILTINS(), name, &v); + LOAD_SP(); + if (err < 0) { goto error; } if (v == NULL) { + SAVE_SP(); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); + LOAD_SP(); goto error; } } @@ -4728,7 +4907,9 @@ int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); @@ -4739,15 +4920,18 @@ self = stack_pointer[-1]; { if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + SAVE_SP(); PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); + LOAD_SP(); if (err) goto pop_3_error; } // we make no attempt to optimize here; specializations should // handle any case whose performance we care about PyObject *stack[] = {class, self}; + SAVE_SP(); PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; @@ -4765,13 +4949,16 @@ } } } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + LOAD_SP(); + INTERPRETER_DECREF(global_super); + INTERPRETER_DECREF(class); + INTERPRETER_DECREF(self); if (super == NULL) goto pop_3_error; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + SAVE_SP(); attr = PyObject_GetAttr(super, name); - Py_DECREF(super); + LOAD_SP(); + INTERPRETER_DECREF(super); if (attr == NULL) goto pop_3_error; null = NULL; } @@ -4799,10 +4986,12 @@ DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + SAVE_SP(); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); + LOAD_SP(); + INTERPRETER_DECREF(global_super); + INTERPRETER_DECREF(class); + INTERPRETER_DECREF(self); if (attr == NULL) goto pop_3_error; stack_pointer[-3] = attr; stack_pointer += -2; @@ -4830,18 +5019,20 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; + SAVE_SP(); attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); + LOAD_SP(); + INTERPRETER_DECREF(global_super); + INTERPRETER_DECREF(class); if (attr == NULL) { - Py_DECREF(self); + INTERPRETER_DECREF(self); if (true) goto pop_3_error; } if (method_found) { self_or_null = self; // transfer ownership } else { - Py_DECREF(self); + INTERPRETER_DECREF(self); self_or_null = NULL; } stack_pointer[-3] = attr; @@ -4872,9 +5063,11 @@ PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; + SAVE_SP(); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - Py_DECREF(codeobj); + LOAD_SP(); + INTERPRETER_DECREF(codeobj); if (func_obj == NULL) { goto error; } @@ -4898,7 +5091,10 @@ assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references - if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; + SAVE_SP(); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, key, value); + LOAD_SP(); + if (err != 0) goto pop_2_error; stack_pointer += -2; DISPATCH(); } @@ -4917,10 +5113,12 @@ // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); + SAVE_SP(); attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); + LOAD_SP(); + INTERPRETER_DECREF(subject); + INTERPRETER_DECREF(type); + INTERPRETER_DECREF(names); if (attrs) { assert(PyTuple_CheckExact(attrs)); // Success! } @@ -4944,7 +5142,9 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). + SAVE_SP(); values_or_none = _PyEval_MatchKeys(tstate, subject, keys); + LOAD_SP(); if (values_or_none == NULL) goto error; stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -4993,7 +5193,11 @@ PyObject *exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; - Py_XSETREF(exc_info->exc_value, exc_value == Py_None ? NULL : exc_value); + PyObject *tmp = exc_info->exc_value; + exc_info->exc_value = exc_value == Py_None ? NULL : exc_value; + if (tmp != NULL) { + INTERPRETER_DECREF(tmp); + } stack_pointer += -1; DISPATCH(); } @@ -5033,7 +5237,7 @@ } else { b = Py_False; - Py_DECREF(value); + INTERPRETER_DECREF(value); } } // _POP_JUMP_IF_TRUE @@ -5067,7 +5271,7 @@ } else { b = Py_False; - Py_DECREF(value); + INTERPRETER_DECREF(value); } } // _POP_JUMP_IF_FALSE @@ -5108,7 +5312,7 @@ INSTRUCTION_STATS(POP_TOP); PyObject *value; value = stack_pointer[-1]; - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer += -1; DISPATCH(); } @@ -5154,6 +5358,7 @@ PyObject **args; args = &stack_pointer[-oparg]; PyObject *cause = NULL, *exc = NULL; + SAVE_SP(); switch (oparg) { case 2: cause = args[1]; @@ -5165,6 +5370,7 @@ if (do_raise(tstate, exc, cause)) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } break; @@ -5173,6 +5379,7 @@ "bad RAISE_VARARGS oparg"); break; } + LOAD_SP(); if (true) { stack_pointer += -oparg; goto error; } } @@ -5201,7 +5408,9 @@ assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); + SAVE_SP(); monitor_reraise(tstate, frame, this_instr); + LOAD_SP(); goto exception_unwind; } @@ -5373,7 +5582,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_Send(receiver, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); @@ -5410,9 +5621,14 @@ if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { + SAVE_SP(); monitor_raise(tstate, frame, this_instr); + LOAD_SP(); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { + SAVE_SP(); + int err = _PyGen_FetchStopIterationValue(&retval); + LOAD_SP(); + if (err == 0) { assert(retval != NULL); JUMPBY(oparg); } @@ -5420,7 +5636,7 @@ goto error; } } - Py_DECREF(v); + INTERPRETER_DECREF(v); } stack_pointer[-1] = retval; DISPATCH(); @@ -5459,22 +5675,27 @@ int err; PyObject *ann_dict; if (LOCALS() == NULL) { + SAVE_SP(); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); + LOAD_SP(); if (true) goto error; } /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) goto error; + SAVE_SP(); + err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + LOAD_SP(); + if (err < 0) goto error; if (ann_dict == NULL) { ann_dict = PyDict_New(); if (ann_dict == NULL) goto error; err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); - Py_DECREF(ann_dict); + INTERPRETER_DECREF(ann_dict); if (err) goto error; } else { - Py_DECREF(ann_dict); + INTERPRETER_DECREF(ann_dict); } DISPATCH(); } @@ -5487,8 +5708,10 @@ PyObject *set; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); int err = PySet_Add(set, v); - Py_DECREF(v); + LOAD_SP(); + INTERPRETER_DECREF(v); if (err) goto pop_1_error; stack_pointer += -1; DISPATCH(); @@ -5539,8 +5762,10 @@ PyObject *set; iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + SAVE_SP(); int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); + LOAD_SP(); + INTERPRETER_DECREF(iterable); if (err < 0) goto pop_1_error; stack_pointer += -1; DISPATCH(); @@ -5564,7 +5789,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; + SAVE_SP(); _Py_Specialize_StoreAttr(owner, next_instr, name); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); @@ -5579,8 +5806,8 @@ SAVE_SP(); int err = PyObject_SetAttr(owner, name, v); LOAD_SP(); - Py_DECREF(v); - Py_DECREF(owner); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(owner); if (err) goto pop_2_error; } stack_pointer += -2; @@ -5623,9 +5850,9 @@ _PyDictValues_AddToInsertionOrder(values, index); } else { - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); } - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } stack_pointer += -2; DISPATCH(); @@ -5655,8 +5882,10 @@ STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; *(PyObject **)addr = value; - Py_XDECREF(old_value); - Py_DECREF(owner); + if (old_value != NULL) { + INTERPRETER_DECREF(old_value); + } + INTERPRETER_DECREF(owner); } stack_pointer += -2; DISPATCH(); @@ -5706,7 +5935,7 @@ new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); STAT_INC(STORE_ATTR, hit); /* Ensure dict is GC tracked if it needs to be */ if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) { @@ -5714,7 +5943,7 @@ } /* PEP 509 */ dict->ma_version_tag = new_version; - Py_DECREF(owner); + INTERPRETER_DECREF(owner); } stack_pointer += -2; DISPATCH(); @@ -5727,7 +5956,9 @@ PyObject *v; v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)GETLOCAL(oparg); + SAVE_SP(); PyCell_SetTakeRef(cell, v); + LOAD_SP(); stack_pointer += -1; DISPATCH(); } @@ -5782,8 +6013,10 @@ PyObject *v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + SAVE_SP(); int err = PyDict_SetItem(GLOBALS(), name, v); - Py_DECREF(v); + LOAD_SP(); + INTERPRETER_DECREF(v); if (err) goto pop_1_error; stack_pointer += -1; DISPATCH(); @@ -5801,7 +6034,7 @@ if (ns == NULL) { _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - Py_DECREF(v); + INTERPRETER_DECREF(v); if (true) goto pop_1_error; } SAVE_SP(); @@ -5810,7 +6043,7 @@ else err = PyObject_SetItem(ns, name, v); LOAD_SP(); - Py_DECREF(v); + INTERPRETER_DECREF(v); if (err) goto pop_1_error; stack_pointer += -1; DISPATCH(); @@ -5837,10 +6070,10 @@ SAVE_SP(); err = PyObject_SetItem(container, slice, v); LOAD_SP(); - Py_DECREF(slice); + INTERPRETER_DECREF(slice); } - Py_DECREF(v); - Py_DECREF(container); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(container); if (err) goto pop_4_error; stack_pointer += -4; DISPATCH(); @@ -5865,7 +6098,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_StoreSubscr(container, sub, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); @@ -5879,9 +6114,9 @@ SAVE_SP(); int err = PyObject_SetItem(container, sub, v); LOAD_SP(); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); + INTERPRETER_DECREF(v); + INTERPRETER_DECREF(container); + INTERPRETER_DECREF(sub); if (err) goto pop_3_error; } stack_pointer += -3; @@ -5902,8 +6137,10 @@ value = stack_pointer[-3]; DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); + SAVE_SP(); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); - Py_DECREF(dict); + LOAD_SP(); + INTERPRETER_DECREF(dict); if (err) goto pop_3_error; stack_pointer += -3; DISPATCH(); @@ -5932,9 +6169,9 @@ PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); assert(old_value != NULL); - Py_DECREF(old_value); + INTERPRETER_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); + INTERPRETER_DECREF(list); stack_pointer += -3; DISPATCH(); } @@ -5970,7 +6207,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_ToBool(value, next_instr); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); @@ -5983,7 +6222,7 @@ SAVE_SP(); int err = PyObject_IsTrue(value); LOAD_SP(); - Py_DECREF(value); + INTERPRETER_DECREF(value); if (err < 0) goto pop_1_error; res = err ? Py_True : Py_False; } @@ -6011,7 +6250,7 @@ // _REPLACE_WITH_TRUE value = owner; { - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; } stack_pointer[-1] = res; @@ -6049,7 +6288,7 @@ res = Py_False; } else { - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; } stack_pointer[-1] = res; @@ -6069,7 +6308,7 @@ DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; - Py_DECREF(value); + INTERPRETER_DECREF(value); stack_pointer[-1] = res; DISPATCH(); } @@ -6110,7 +6349,7 @@ } else { assert(Py_SIZE(value)); - Py_DECREF(value); + INTERPRETER_DECREF(value); res = Py_True; } stack_pointer[-1] = res; @@ -6124,8 +6363,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; + SAVE_SP(); res = PyNumber_Invert(value); - Py_DECREF(value); + LOAD_SP(); + INTERPRETER_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); @@ -6141,7 +6382,7 @@ SAVE_SP(); res = PyNumber_Negative(value); LOAD_SP(); - Py_DECREF(value); + INTERPRETER_DECREF(value); if (res == NULL) goto pop_1_error; stack_pointer[-1] = res; DISPATCH(); @@ -6168,8 +6409,10 @@ seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq); + LOAD_SP(); + INTERPRETER_DECREF(seq); if (res == 0) goto pop_1_error; stack_pointer += (oparg >> 8) + (oparg & 0xFF); DISPATCH(); @@ -6191,7 +6434,9 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + SAVE_SP(); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + LOAD_SP(); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); @@ -6203,8 +6448,10 @@ // _UNPACK_SEQUENCE { PyObject **top = stack_pointer + oparg - 1; + SAVE_SP(); int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); + LOAD_SP(); + INTERPRETER_DECREF(seq); if (res == 0) goto pop_1_error; } stack_pointer += -1 + oparg; @@ -6228,7 +6475,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer += -1 + oparg; DISPATCH(); } @@ -6250,7 +6497,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = Py_NewRef(items[i]); } - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer += -1 + oparg; DISPATCH(); } @@ -6271,7 +6518,7 @@ STAT_INC(UNPACK_SEQUENCE, hit); val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); - Py_DECREF(seq); + INTERPRETER_DECREF(seq); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -6305,7 +6552,7 @@ tb = Py_None; } else { - Py_DECREF(tb); + INTERPRETER_DECREF(tb); } assert(PyLong_Check(lasti)); (void)lasti; // Shut up compiler warning if asserts are off diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index cc9eb8a0e90eeb..988c27d9582dc3 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -130,15 +130,39 @@ def replace_decrefs( continue if var.size != "1": out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"Py_DECREF({var.name}[_i]);\n") + out.emit(f"INTERPRETER_DECREF({var.name}[_i]);\n") out.emit("}\n") elif var.condition: if var.condition == "1": - out.emit(f"Py_DECREF({var.name});\n") + out.emit(f"INTERPRETER_DECREF({var.name});\n") elif var.condition != "0": - out.emit(f"Py_XDECREF({var.name});\n") + out.emit(f"if ({var.name} != NULL) {{ INTERPRETER_DECREF({var.name}); }}\n") else: - out.emit(f"Py_DECREF({var.name});\n") + out.emit(f"INTERPRETER_DECREF({var.name});\n") + +def replace_decref( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + out.emit_at("INTERPRETER_DECREF", tkn) + out.emit(next(tkn_iter)) + emit_to(out, tkn_iter, "RPAREN") + next(tkn_iter) # Semi colon + out.emit(");\n") + +def replace_xdecref( + out: CWriter, + tkn: Token, + tkn_iter: Iterator[Token], + uop: Uop, + unused: Stack, + inst: Instruction | None, +) -> None: + raise Exception("XDECREF not supported") def replace_sync_sp( @@ -175,6 +199,8 @@ def replace_check_eval_breaker( "DEOPT_IF": replace_deopt, "ERROR_IF": replace_error, "ERROR_NO_POP": replace_error_no_pop, + "Py_DECREF": replace_decref, + "Py_XDECREF": replace_xdecref, "DECREF_INPUTS": replace_decrefs, "CHECK_EVAL_BREAKER": replace_check_eval_breaker, "SYNC_SP": replace_sync_sp,