diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index cf830b6066f4ab..1728c576f5cc41 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -29,7 +29,7 @@ # error "this header file must not be included directly" #endif -#define PYSTATS_MAX_UOP_ID 1024 +#define PYSTATS_MAX_UOP_ID 1500 #define SPECIALIZATION_FAILURE_KINDS 60 diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index fa9568ab4d0e85..7ccf56acd871cd 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -15,7 +15,6 @@ extern "C" { #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_typedefs.h" // _PyRuntimeState - #define CODE_MAX_WATCHERS 8 #define CONTEXT_MAX_WATCHERS 8 #define FUNC_MAX_WATCHERS 8 diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 8a88cbf607ba4b..ba8790213c4e52 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -15,7 +15,7 @@ extern "C" { #ifdef _Py_JIT -typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2); int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bd6b84ec7fd908..e0803189415e0e 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1028,9 +1028,10 @@ enum InstructionFormat { #define HAS_ESCAPES_FLAG (512) #define HAS_EXIT_FLAG (1024) #define HAS_PURE_FLAG (2048) -#define HAS_ERROR_NO_POP_FLAG (4096) -#define HAS_NO_SAVE_IP_FLAG (8192) -#define HAS_PERIODIC_FLAG (16384) +#define HAS_SYNC_SP_FLAG (4096) +#define HAS_ERROR_NO_POP_FLAG (8192) +#define HAS_NO_SAVE_IP_FLAG (16384) +#define HAS_PERIODIC_FLAG (32768) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1043,6 +1044,7 @@ enum InstructionFormat { #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) #define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) +#define OPCODE_HAS_SYNC_SP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_SYNC_SP_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) #define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) @@ -1077,7 +1079,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_SYNC_SP_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1094,22 +1096,22 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BUILD_TEMPLATE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1117,8 +1119,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, @@ -1152,7 +1154,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_SYNC_SP_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, @@ -1164,9 +1166,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1174,7 +1176,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, @@ -1197,7 +1199,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_SYNC_SP_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, @@ -1205,7 +1207,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1255,8 +1257,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_SYNC_SP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 9f930f2107ed5e..bd42e3d4d6b3c4 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -35,7 +35,7 @@ typedef struct { uint8_t linked:1; uint8_t chain_depth:6; // Must be big enough for MAX_CHAIN_DEPTH - 1. bool warm; - int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). + int16_t index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). _PyBloomFilter bloom; _PyExecutorLinkListNode links; PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). @@ -119,7 +119,7 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #define JIT_CLEANUP_THRESHOLD 100000 // This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 800 +#define UOP_MAX_TRACE_LENGTH 1600 #define TRACE_STACK_SIZE 5 @@ -373,7 +373,7 @@ static inline int is_terminator(const _PyUOpInstruction *uop) ); } -extern void _PyExecutor_Free(_PyExecutorObject *self); +extern void _PyExecutor_FreeColdExecutor(_PyExecutorObject *cold); PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); #ifdef _Py_TIER2 diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index c4e8f10fe05276..d856fc839c32ac 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -70,6 +70,9 @@ static const _PyStackRef PyStackRef_ERROR = { .index = 2 }; #define PyStackRef_False ((_PyStackRef){ .index = 6 }) #define PyStackRef_True ((_PyStackRef){ .index = 8 }) +#define PyStackRef_ZERO_BITS PyStackRef_NULL + + #define INITIAL_STACKREF_INDEX 10 static inline int @@ -264,6 +267,10 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID }; +/* For use in the JIT to clear an unused value. + * PyStackRef_ZERO_BITS has no meaning and should not be used other than by the JIT. */ +static const _PyStackRef PyStackRef_ZERO_BITS = { .bits = 0 }; + /* Wrap a pointer in a stack ref. * The resulting stack reference is not safe and should only be used * in the interpreter to pass values from one uop to another. @@ -303,6 +310,16 @@ PyStackRef_IsValid(_PyStackRef ref) return ref.bits >= Py_INT_TAG; } +static inline bool +PyStackRef_IsWrapped(_PyStackRef ref) +{ +#ifdef Py_DEBUG + return (ref.bits & Py_TAG_INVALID) != 0 && !PyStackRef_IsError(ref); +#else + Py_FatalError("Cannot determine if value is wrapped if Py_DEBUG is not set"); +#endif +} + static inline bool PyStackRef_IsTaggedInt(_PyStackRef i) { @@ -810,6 +827,17 @@ _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) PyStackRef_XCLOSE(ref->ref); } +static inline _PyStackRef +_PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + assert(tstate_impl->c_stack_refs == ref); + tstate_impl->c_stack_refs = ref->next; +#endif + return ref->ref; +} + #ifdef Py_GIL_DISABLED static inline int diff --git a/Include/internal/pycore_stats.h b/Include/internal/pycore_stats.h index ab649574f33dbf..51ceea87ba3a15 100644 --- a/Include/internal/pycore_stats.h +++ b/Include/internal/pycore_stats.h @@ -28,7 +28,7 @@ extern "C" { #define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0) #define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) #define OPT_STAT_ADD(name, n) do { if (_Py_stats) _Py_stats->optimization_stats.name += (n); } while (0) -#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) +#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) #define UOP_PAIR_INC(uopcode, lastuop) \ do { \ if (lastuop && _Py_stats) { \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 749369a40aecdd..f955254dc2ac8c 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -242,9 +242,7 @@ extern "C" { #define _LOAD_FAST_BORROW_5 477 #define _LOAD_FAST_BORROW_6 478 #define _LOAD_FAST_BORROW_7 479 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK -#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS #define _LOAD_GLOBAL 480 @@ -313,50 +311,689 @@ extern "C" { #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 522 -#define _STORE_ATTR 523 -#define _STORE_ATTR_INSTANCE_VALUE 524 -#define _STORE_ATTR_SLOT 525 -#define _STORE_ATTR_WITH_HINT 526 +#define _SPILL_OR_RELOAD 522 +#define _START_EXECUTOR 523 +#define _STORE_ATTR 524 +#define _STORE_ATTR_INSTANCE_VALUE 525 +#define _STORE_ATTR_SLOT 526 +#define _STORE_ATTR_WITH_HINT 527 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 527 -#define _STORE_FAST_0 528 -#define _STORE_FAST_1 529 -#define _STORE_FAST_2 530 -#define _STORE_FAST_3 531 -#define _STORE_FAST_4 532 -#define _STORE_FAST_5 533 -#define _STORE_FAST_6 534 -#define _STORE_FAST_7 535 -#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST -#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST +#define _STORE_FAST 528 +#define _STORE_FAST_0 529 +#define _STORE_FAST_1 530 +#define _STORE_FAST_2 531 +#define _STORE_FAST_3 532 +#define _STORE_FAST_4 533 +#define _STORE_FAST_5 534 +#define _STORE_FAST_6 535 +#define _STORE_FAST_7 536 #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 536 -#define _STORE_SUBSCR 537 -#define _STORE_SUBSCR_DICT 538 -#define _STORE_SUBSCR_LIST_INT 539 -#define _SWAP 540 -#define _SWAP_2 541 -#define _SWAP_3 542 -#define _TIER2_RESUME_CHECK 543 -#define _TO_BOOL 544 +#define _STORE_SLICE 537 +#define _STORE_SUBSCR 538 +#define _STORE_SUBSCR_DICT 539 +#define _STORE_SUBSCR_LIST_INT 540 +#define _SWAP 541 +#define _SWAP_2 542 +#define _SWAP_3 543 +#define _TIER2_RESUME_CHECK 544 +#define _TO_BOOL 545 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 545 +#define _TO_BOOL_LIST 546 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 546 +#define _TO_BOOL_STR 547 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 547 -#define _UNPACK_SEQUENCE_LIST 548 -#define _UNPACK_SEQUENCE_TUPLE 549 -#define _UNPACK_SEQUENCE_TWO_TUPLE 550 +#define _UNPACK_SEQUENCE 548 +#define _UNPACK_SEQUENCE_LIST 549 +#define _UNPACK_SEQUENCE_TUPLE 550 +#define _UNPACK_SEQUENCE_TWO_TUPLE 551 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 550 +#define MAX_UOP_ID 551 +#define _BINARY_OP_r21 552 +#define _BINARY_OP_ADD_FLOAT_r21 553 +#define _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21 554 +#define _BINARY_OP_ADD_INT_r21 555 +#define _BINARY_OP_ADD_UNICODE_r21 556 +#define _BINARY_OP_EXTEND_r21 557 +#define _BINARY_OP_INPLACE_ADD_UNICODE_r20 558 +#define _BINARY_OP_MULTIPLY_FLOAT_r21 559 +#define _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21 560 +#define _BINARY_OP_MULTIPLY_INT_r21 561 +#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 562 +#define _BINARY_OP_SUBSCR_DICT_r21 563 +#define _BINARY_OP_SUBSCR_INIT_CALL_r31 564 +#define _BINARY_OP_SUBSCR_LIST_INT_r21 565 +#define _BINARY_OP_SUBSCR_LIST_SLICE_r21 566 +#define _BINARY_OP_SUBSCR_STR_INT_r21 567 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r21 568 +#define _BINARY_OP_SUBTRACT_FLOAT_r21 569 +#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21 570 +#define _BINARY_OP_SUBTRACT_INT_r21 571 +#define _BINARY_SLICE_r31 572 +#define _BUILD_INTERPOLATION_r01 573 +#define _BUILD_LIST_r01 574 +#define _BUILD_MAP_r01 575 +#define _BUILD_SET_r01 576 +#define _BUILD_SLICE_r01 577 +#define _BUILD_STRING_r01 578 +#define _BUILD_TEMPLATE_r21 579 +#define _BUILD_TUPLE_r01 580 +#define _CALL_BUILTIN_CLASS_r01 581 +#define _CALL_BUILTIN_FAST_r01 582 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 583 +#define _CALL_BUILTIN_O_r01 584 +#define _CALL_INTRINSIC_1_r11 585 +#define _CALL_INTRINSIC_2_r21 586 +#define _CALL_ISINSTANCE_r31 587 +#define _CALL_KW_NON_PY_r11 588 +#define _CALL_LEN_r31 589 +#define _CALL_LIST_APPEND_r30 590 +#define _CALL_METHOD_DESCRIPTOR_FAST_r01 591 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 592 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_r01 593 +#define _CALL_METHOD_DESCRIPTOR_O_r01 594 +#define _CALL_NON_PY_GENERAL_r01 595 +#define _CALL_STR_1_r31 596 +#define _CALL_TUPLE_1_r31 597 +#define _CALL_TYPE_1_r31 598 +#define _CHECK_AND_ALLOCATE_OBJECT_r00 599 +#define _CHECK_ATTR_CLASS_r00 600 +#define _CHECK_ATTR_CLASS_r11 601 +#define _CHECK_ATTR_CLASS_r22 602 +#define _CHECK_ATTR_CLASS_r33 603 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r00 604 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 605 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 606 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 607 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 608 +#define _CHECK_EG_MATCH_r22 609 +#define _CHECK_EXC_MATCH_r22 610 +#define _CHECK_FUNCTION_r00 611 +#define _CHECK_FUNCTION_r11 612 +#define _CHECK_FUNCTION_r22 613 +#define _CHECK_FUNCTION_r33 614 +#define _CHECK_FUNCTION_EXACT_ARGS_r00 615 +#define _CHECK_FUNCTION_VERSION_r00 616 +#define _CHECK_FUNCTION_VERSION_INLINE_r00 617 +#define _CHECK_FUNCTION_VERSION_INLINE_r11 618 +#define _CHECK_FUNCTION_VERSION_INLINE_r22 619 +#define _CHECK_FUNCTION_VERSION_INLINE_r33 620 +#define _CHECK_FUNCTION_VERSION_KW_r11 621 +#define _CHECK_IS_NOT_PY_CALLABLE_r00 622 +#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 623 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r00 624 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 625 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 626 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 627 +#define _CHECK_METHOD_VERSION_r00 628 +#define _CHECK_METHOD_VERSION_KW_r11 629 +#define _CHECK_PEP_523_r00 630 +#define _CHECK_PEP_523_r11 631 +#define _CHECK_PEP_523_r22 632 +#define _CHECK_PEP_523_r33 633 +#define _CHECK_PERIODIC_r00 634 +#define _CHECK_PERIODIC_AT_END_r00 635 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 636 +#define _CHECK_RECURSION_REMAINING_r00 637 +#define _CHECK_RECURSION_REMAINING_r11 638 +#define _CHECK_RECURSION_REMAINING_r22 639 +#define _CHECK_RECURSION_REMAINING_r33 640 +#define _CHECK_STACK_SPACE_r00 641 +#define _CHECK_STACK_SPACE_OPERAND_r00 642 +#define _CHECK_STACK_SPACE_OPERAND_r11 643 +#define _CHECK_STACK_SPACE_OPERAND_r22 644 +#define _CHECK_STACK_SPACE_OPERAND_r33 645 +#define _CHECK_VALIDITY_r00 646 +#define _CHECK_VALIDITY_r11 647 +#define _CHECK_VALIDITY_r22 648 +#define _CHECK_VALIDITY_r33 649 +#define _COLD_EXIT_r00 650 +#define _COMPARE_OP_r21 651 +#define _COMPARE_OP_FLOAT_r21 652 +#define _COMPARE_OP_INT_r21 653 +#define _COMPARE_OP_STR_r21 654 +#define _CONTAINS_OP_r21 655 +#define _CONTAINS_OP_DICT_r21 656 +#define _CONTAINS_OP_SET_r21 657 +#define _CONVERT_VALUE_r11 658 +#define _COPY_r01 659 +#define _COPY_1_r01 660 +#define _COPY_1_r12 661 +#define _COPY_1_r23 662 +#define _COPY_2_r23 663 +#define _COPY_3_r33 664 +#define _COPY_FREE_VARS_r00 665 +#define _CREATE_INIT_FRAME_r01 666 +#define _DELETE_ATTR_r10 667 +#define _DELETE_DEREF_r00 668 +#define _DELETE_FAST_r00 669 +#define _DELETE_GLOBAL_r00 670 +#define _DELETE_NAME_r00 671 +#define _DELETE_SUBSCR_r20 672 +#define _DEOPT_r00 673 +#define _DEOPT_r10 674 +#define _DEOPT_r20 675 +#define _DEOPT_r30 676 +#define _DICT_MERGE_r10 677 +#define _DICT_UPDATE_r10 678 +#define _DO_CALL_r01 679 +#define _DO_CALL_FUNCTION_EX_r31 680 +#define _DO_CALL_KW_r11 681 +#define _END_FOR_r10 682 +#define _END_SEND_r21 683 +#define _ERROR_POP_N_r00 684 +#define _EXIT_INIT_CHECK_r10 685 +#define _EXIT_TRACE_r00 686 +#define _EXIT_TRACE_r10 687 +#define _EXIT_TRACE_r20 688 +#define _EXIT_TRACE_r30 689 +#define _EXPAND_METHOD_r00 690 +#define _EXPAND_METHOD_KW_r11 691 +#define _FATAL_ERROR_r00 692 +#define _FATAL_ERROR_r11 693 +#define _FATAL_ERROR_r22 694 +#define _FATAL_ERROR_r33 695 +#define _FORMAT_SIMPLE_r11 696 +#define _FORMAT_WITH_SPEC_r21 697 +#define _FOR_ITER_r23 698 +#define _FOR_ITER_GEN_FRAME_r23 699 +#define _FOR_ITER_TIER_TWO_r23 700 +#define _GET_AITER_r11 701 +#define _GET_ANEXT_r12 702 +#define _GET_AWAITABLE_r11 703 +#define _GET_ITER_r12 704 +#define _GET_LEN_r12 705 +#define _GET_YIELD_FROM_ITER_r11 706 +#define _GUARD_BINARY_OP_EXTEND_r22 707 +#define _GUARD_CALLABLE_ISINSTANCE_r33 708 +#define _GUARD_CALLABLE_LEN_r33 709 +#define _GUARD_CALLABLE_LIST_APPEND_r33 710 +#define _GUARD_CALLABLE_STR_1_r33 711 +#define _GUARD_CALLABLE_TUPLE_1_r33 712 +#define _GUARD_CALLABLE_TYPE_1_r33 713 +#define _GUARD_DORV_NO_DICT_r00 714 +#define _GUARD_DORV_NO_DICT_r11 715 +#define _GUARD_DORV_NO_DICT_r22 716 +#define _GUARD_DORV_NO_DICT_r33 717 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00 718 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 719 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 720 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 721 +#define _GUARD_GLOBALS_VERSION_r00 722 +#define _GUARD_GLOBALS_VERSION_r11 723 +#define _GUARD_GLOBALS_VERSION_r22 724 +#define _GUARD_GLOBALS_VERSION_r33 725 +#define _GUARD_IS_FALSE_POP_r10 726 +#define _GUARD_IS_FALSE_POP_r21 727 +#define _GUARD_IS_FALSE_POP_r32 728 +#define _GUARD_IS_NONE_POP_r10 729 +#define _GUARD_IS_NONE_POP_r21 730 +#define _GUARD_IS_NONE_POP_r32 731 +#define _GUARD_IS_NOT_NONE_POP_r10 732 +#define _GUARD_IS_TRUE_POP_r10 733 +#define _GUARD_IS_TRUE_POP_r21 734 +#define _GUARD_IS_TRUE_POP_r32 735 +#define _GUARD_KEYS_VERSION_r00 736 +#define _GUARD_KEYS_VERSION_r11 737 +#define _GUARD_KEYS_VERSION_r22 738 +#define _GUARD_KEYS_VERSION_r33 739 +#define _GUARD_NOS_DICT_r00 740 +#define _GUARD_NOS_DICT_r11 741 +#define _GUARD_NOS_DICT_r22 742 +#define _GUARD_NOS_DICT_r33 743 +#define _GUARD_NOS_FLOAT_r00 744 +#define _GUARD_NOS_FLOAT_r11 745 +#define _GUARD_NOS_FLOAT_r22 746 +#define _GUARD_NOS_FLOAT_r33 747 +#define _GUARD_NOS_INT_r00 748 +#define _GUARD_NOS_INT_r11 749 +#define _GUARD_NOS_INT_r22 750 +#define _GUARD_NOS_INT_r33 751 +#define _GUARD_NOS_LIST_r00 752 +#define _GUARD_NOS_LIST_r11 753 +#define _GUARD_NOS_LIST_r22 754 +#define _GUARD_NOS_LIST_r33 755 +#define _GUARD_NOS_NOT_NULL_r00 756 +#define _GUARD_NOS_NOT_NULL_r11 757 +#define _GUARD_NOS_NOT_NULL_r22 758 +#define _GUARD_NOS_NOT_NULL_r33 759 +#define _GUARD_NOS_NULL_r00 760 +#define _GUARD_NOS_NULL_r11 761 +#define _GUARD_NOS_NULL_r22 762 +#define _GUARD_NOS_NULL_r33 763 +#define _GUARD_NOS_OVERFLOWED_r00 764 +#define _GUARD_NOS_OVERFLOWED_r11 765 +#define _GUARD_NOS_OVERFLOWED_r22 766 +#define _GUARD_NOS_OVERFLOWED_r33 767 +#define _GUARD_NOS_TUPLE_r00 768 +#define _GUARD_NOS_TUPLE_r11 769 +#define _GUARD_NOS_TUPLE_r22 770 +#define _GUARD_NOS_TUPLE_r33 771 +#define _GUARD_NOS_UNICODE_r00 772 +#define _GUARD_NOS_UNICODE_r11 773 +#define _GUARD_NOS_UNICODE_r22 774 +#define _GUARD_NOS_UNICODE_r33 775 +#define _GUARD_NOT_EXHAUSTED_LIST_r00 776 +#define _GUARD_NOT_EXHAUSTED_LIST_r11 777 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 778 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 779 +#define _GUARD_NOT_EXHAUSTED_RANGE_r00 780 +#define _GUARD_NOT_EXHAUSTED_RANGE_r11 781 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 782 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 783 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r00 784 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r11 785 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 786 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 787 +#define _GUARD_THIRD_NULL_r33 788 +#define _GUARD_TOS_ANY_SET_r00 789 +#define _GUARD_TOS_ANY_SET_r11 790 +#define _GUARD_TOS_ANY_SET_r22 791 +#define _GUARD_TOS_ANY_SET_r33 792 +#define _GUARD_TOS_DICT_r00 793 +#define _GUARD_TOS_DICT_r11 794 +#define _GUARD_TOS_DICT_r22 795 +#define _GUARD_TOS_DICT_r33 796 +#define _GUARD_TOS_FLOAT_r00 797 +#define _GUARD_TOS_FLOAT_r11 798 +#define _GUARD_TOS_FLOAT_r22 799 +#define _GUARD_TOS_FLOAT_r33 800 +#define _GUARD_TOS_INT_r00 801 +#define _GUARD_TOS_INT_r11 802 +#define _GUARD_TOS_INT_r22 803 +#define _GUARD_TOS_INT_r33 804 +#define _GUARD_TOS_LIST_r00 805 +#define _GUARD_TOS_LIST_r11 806 +#define _GUARD_TOS_LIST_r22 807 +#define _GUARD_TOS_LIST_r33 808 +#define _GUARD_TOS_OVERFLOWED_r00 809 +#define _GUARD_TOS_OVERFLOWED_r11 810 +#define _GUARD_TOS_OVERFLOWED_r22 811 +#define _GUARD_TOS_OVERFLOWED_r33 812 +#define _GUARD_TOS_SLICE_r00 813 +#define _GUARD_TOS_SLICE_r11 814 +#define _GUARD_TOS_SLICE_r22 815 +#define _GUARD_TOS_SLICE_r33 816 +#define _GUARD_TOS_TUPLE_r00 817 +#define _GUARD_TOS_TUPLE_r11 818 +#define _GUARD_TOS_TUPLE_r22 819 +#define _GUARD_TOS_TUPLE_r33 820 +#define _GUARD_TOS_UNICODE_r00 821 +#define _GUARD_TOS_UNICODE_r11 822 +#define _GUARD_TOS_UNICODE_r22 823 +#define _GUARD_TOS_UNICODE_r33 824 +#define _GUARD_TYPE_VERSION_r00 825 +#define _GUARD_TYPE_VERSION_r11 826 +#define _GUARD_TYPE_VERSION_r22 827 +#define _GUARD_TYPE_VERSION_r33 828 +#define _GUARD_TYPE_VERSION_AND_LOCK_r00 829 +#define _GUARD_TYPE_VERSION_AND_LOCK_r11 830 +#define _GUARD_TYPE_VERSION_AND_LOCK_r22 831 +#define _GUARD_TYPE_VERSION_AND_LOCK_r33 832 +#define _HANDLE_PENDING_AND_DEOPT_r00 833 +#define _HANDLE_PENDING_AND_DEOPT_r10 834 +#define _HANDLE_PENDING_AND_DEOPT_r20 835 +#define _HANDLE_PENDING_AND_DEOPT_r30 836 +#define _IMPORT_FROM_r12 837 +#define _IMPORT_NAME_r21 838 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 839 +#define _INIT_CALL_PY_EXACT_ARGS_r01 840 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 841 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 842 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 843 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 844 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 845 +#define _INSERT_NULL_r10 846 +#define _INSTRUMENTED_FOR_ITER_r23 847 +#define _INSTRUMENTED_INSTRUCTION_r00 848 +#define _INSTRUMENTED_JUMP_FORWARD_r00 849 +#define _INSTRUMENTED_JUMP_FORWARD_r11 850 +#define _INSTRUMENTED_JUMP_FORWARD_r22 851 +#define _INSTRUMENTED_JUMP_FORWARD_r33 852 +#define _INSTRUMENTED_LINE_r00 853 +#define _INSTRUMENTED_NOT_TAKEN_r00 854 +#define _INSTRUMENTED_NOT_TAKEN_r11 855 +#define _INSTRUMENTED_NOT_TAKEN_r22 856 +#define _INSTRUMENTED_NOT_TAKEN_r33 857 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 858 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 859 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 860 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 861 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 862 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 863 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 864 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 865 +#define _IS_NONE_r11 866 +#define _IS_OP_r21 867 +#define _ITER_CHECK_LIST_r00 868 +#define _ITER_CHECK_LIST_r11 869 +#define _ITER_CHECK_LIST_r22 870 +#define _ITER_CHECK_LIST_r33 871 +#define _ITER_CHECK_RANGE_r00 872 +#define _ITER_CHECK_RANGE_r11 873 +#define _ITER_CHECK_RANGE_r22 874 +#define _ITER_CHECK_RANGE_r33 875 +#define _ITER_CHECK_TUPLE_r00 876 +#define _ITER_CHECK_TUPLE_r11 877 +#define _ITER_CHECK_TUPLE_r22 878 +#define _ITER_CHECK_TUPLE_r33 879 +#define _ITER_JUMP_LIST_r00 880 +#define _ITER_JUMP_LIST_r11 881 +#define _ITER_JUMP_LIST_r22 882 +#define _ITER_JUMP_LIST_r33 883 +#define _ITER_JUMP_RANGE_r00 884 +#define _ITER_JUMP_RANGE_r11 885 +#define _ITER_JUMP_RANGE_r22 886 +#define _ITER_JUMP_RANGE_r33 887 +#define _ITER_JUMP_TUPLE_r00 888 +#define _ITER_JUMP_TUPLE_r11 889 +#define _ITER_JUMP_TUPLE_r22 890 +#define _ITER_JUMP_TUPLE_r33 891 +#define _ITER_NEXT_LIST_r23 892 +#define _ITER_NEXT_LIST_TIER_TWO_r23 893 +#define _ITER_NEXT_RANGE_r23 894 +#define _ITER_NEXT_TUPLE_r23 895 +#define _JUMP_TO_TOP_r00 896 +#define _LIST_APPEND_r10 897 +#define _LIST_EXTEND_r10 898 +#define _LOAD_ATTR_r10 899 +#define _LOAD_ATTR_CLASS_r11 900 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 901 +#define _LOAD_ATTR_INSTANCE_VALUE_r11 902 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r01 903 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 904 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 905 +#define _LOAD_ATTR_METHOD_NO_DICT_r01 906 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 907 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 908 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r01 909 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 910 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 911 +#define _LOAD_ATTR_MODULE_r11 912 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 913 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 914 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 915 +#define _LOAD_ATTR_SLOT_r11 916 +#define _LOAD_ATTR_WITH_HINT_r11 917 +#define _LOAD_BUILD_CLASS_r01 918 +#define _LOAD_BYTECODE_r00 919 +#define _LOAD_COMMON_CONSTANT_r01 920 +#define _LOAD_COMMON_CONSTANT_r12 921 +#define _LOAD_COMMON_CONSTANT_r23 922 +#define _LOAD_CONST_r01 923 +#define _LOAD_CONST_r12 924 +#define _LOAD_CONST_r23 925 +#define _LOAD_CONST_INLINE_r01 926 +#define _LOAD_CONST_INLINE_r12 927 +#define _LOAD_CONST_INLINE_r23 928 +#define _LOAD_CONST_INLINE_BORROW_r01 929 +#define _LOAD_CONST_INLINE_BORROW_r12 930 +#define _LOAD_CONST_INLINE_BORROW_r23 931 +#define _LOAD_CONST_UNDER_INLINE_r01 932 +#define _LOAD_CONST_UNDER_INLINE_r12 933 +#define _LOAD_CONST_UNDER_INLINE_r23 934 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r01 935 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 936 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 937 +#define _LOAD_DEREF_r01 938 +#define _LOAD_FAST_r01 939 +#define _LOAD_FAST_r12 940 +#define _LOAD_FAST_r23 941 +#define _LOAD_FAST_0_r01 942 +#define _LOAD_FAST_0_r12 943 +#define _LOAD_FAST_0_r23 944 +#define _LOAD_FAST_1_r01 945 +#define _LOAD_FAST_1_r12 946 +#define _LOAD_FAST_1_r23 947 +#define _LOAD_FAST_2_r01 948 +#define _LOAD_FAST_2_r12 949 +#define _LOAD_FAST_2_r23 950 +#define _LOAD_FAST_3_r01 951 +#define _LOAD_FAST_3_r12 952 +#define _LOAD_FAST_3_r23 953 +#define _LOAD_FAST_4_r01 954 +#define _LOAD_FAST_4_r12 955 +#define _LOAD_FAST_4_r23 956 +#define _LOAD_FAST_5_r01 957 +#define _LOAD_FAST_5_r12 958 +#define _LOAD_FAST_5_r23 959 +#define _LOAD_FAST_6_r01 960 +#define _LOAD_FAST_6_r12 961 +#define _LOAD_FAST_6_r23 962 +#define _LOAD_FAST_7_r01 963 +#define _LOAD_FAST_7_r12 964 +#define _LOAD_FAST_7_r23 965 +#define _LOAD_FAST_AND_CLEAR_r01 966 +#define _LOAD_FAST_AND_CLEAR_r12 967 +#define _LOAD_FAST_AND_CLEAR_r23 968 +#define _LOAD_FAST_BORROW_r01 969 +#define _LOAD_FAST_BORROW_r12 970 +#define _LOAD_FAST_BORROW_r23 971 +#define _LOAD_FAST_BORROW_0_r01 972 +#define _LOAD_FAST_BORROW_0_r12 973 +#define _LOAD_FAST_BORROW_0_r23 974 +#define _LOAD_FAST_BORROW_1_r01 975 +#define _LOAD_FAST_BORROW_1_r12 976 +#define _LOAD_FAST_BORROW_1_r23 977 +#define _LOAD_FAST_BORROW_2_r01 978 +#define _LOAD_FAST_BORROW_2_r12 979 +#define _LOAD_FAST_BORROW_2_r23 980 +#define _LOAD_FAST_BORROW_3_r01 981 +#define _LOAD_FAST_BORROW_3_r12 982 +#define _LOAD_FAST_BORROW_3_r23 983 +#define _LOAD_FAST_BORROW_4_r01 984 +#define _LOAD_FAST_BORROW_4_r12 985 +#define _LOAD_FAST_BORROW_4_r23 986 +#define _LOAD_FAST_BORROW_5_r01 987 +#define _LOAD_FAST_BORROW_5_r12 988 +#define _LOAD_FAST_BORROW_5_r23 989 +#define _LOAD_FAST_BORROW_6_r01 990 +#define _LOAD_FAST_BORROW_6_r12 991 +#define _LOAD_FAST_BORROW_6_r23 992 +#define _LOAD_FAST_BORROW_7_r01 993 +#define _LOAD_FAST_BORROW_7_r12 994 +#define _LOAD_FAST_BORROW_7_r23 995 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 996 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 997 +#define _LOAD_FAST_CHECK_r01 998 +#define _LOAD_FAST_LOAD_FAST_r02 999 +#define _LOAD_FAST_LOAD_FAST_r13 1000 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1001 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1002 +#define _LOAD_GLOBAL_r00 1003 +#define _LOAD_GLOBAL_BUILTINS_r01 1004 +#define _LOAD_GLOBAL_MODULE_r01 1005 +#define _LOAD_LOCALS_r01 1006 +#define _LOAD_NAME_r01 1007 +#define _LOAD_SMALL_INT_r01 1008 +#define _LOAD_SMALL_INT_r12 1009 +#define _LOAD_SMALL_INT_r23 1010 +#define _LOAD_SMALL_INT_0_r01 1011 +#define _LOAD_SMALL_INT_0_r12 1012 +#define _LOAD_SMALL_INT_0_r23 1013 +#define _LOAD_SMALL_INT_1_r01 1014 +#define _LOAD_SMALL_INT_1_r12 1015 +#define _LOAD_SMALL_INT_1_r23 1016 +#define _LOAD_SMALL_INT_2_r01 1017 +#define _LOAD_SMALL_INT_2_r12 1018 +#define _LOAD_SMALL_INT_2_r23 1019 +#define _LOAD_SMALL_INT_3_r01 1020 +#define _LOAD_SMALL_INT_3_r12 1021 +#define _LOAD_SMALL_INT_3_r23 1022 +#define _LOAD_SPECIAL_r00 1023 +#define _LOAD_SUPER_ATTR_ATTR_r31 1024 +#define _LOAD_SUPER_ATTR_METHOD_r32 1025 +#define _MAKE_CALLARGS_A_TUPLE_r33 1026 +#define _MAKE_CELL_r00 1027 +#define _MAKE_FUNCTION_r11 1028 +#define _MAKE_WARM_r00 1029 +#define _MAKE_WARM_r11 1030 +#define _MAKE_WARM_r22 1031 +#define _MAKE_WARM_r33 1032 +#define _MAP_ADD_r20 1033 +#define _MATCH_CLASS_r31 1034 +#define _MATCH_KEYS_r23 1035 +#define _MATCH_MAPPING_r01 1036 +#define _MATCH_MAPPING_r12 1037 +#define _MATCH_MAPPING_r23 1038 +#define _MATCH_SEQUENCE_r01 1039 +#define _MATCH_SEQUENCE_r12 1040 +#define _MATCH_SEQUENCE_r23 1041 +#define _MAYBE_EXPAND_METHOD_r00 1042 +#define _MAYBE_EXPAND_METHOD_KW_r11 1043 +#define _MONITOR_CALL_r00 1044 +#define _MONITOR_CALL_KW_r11 1045 +#define _MONITOR_JUMP_BACKWARD_r00 1046 +#define _MONITOR_JUMP_BACKWARD_r11 1047 +#define _MONITOR_JUMP_BACKWARD_r22 1048 +#define _MONITOR_JUMP_BACKWARD_r33 1049 +#define _MONITOR_RESUME_r00 1050 +#define _NOP_r00 1051 +#define _NOP_r11 1052 +#define _NOP_r22 1053 +#define _NOP_r33 1054 +#define _POP_CALL_r20 1055 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1056 +#define _POP_CALL_ONE_r30 1057 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1058 +#define _POP_CALL_TWO_r30 1059 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1060 +#define _POP_EXCEPT_r10 1061 +#define _POP_ITER_r20 1062 +#define _POP_JUMP_IF_FALSE_r10 1063 +#define _POP_JUMP_IF_FALSE_r21 1064 +#define _POP_JUMP_IF_FALSE_r32 1065 +#define _POP_JUMP_IF_TRUE_r10 1066 +#define _POP_JUMP_IF_TRUE_r21 1067 +#define _POP_JUMP_IF_TRUE_r32 1068 +#define _POP_TOP_r10 1069 +#define _POP_TOP_FLOAT_r10 1070 +#define _POP_TOP_FLOAT_r21 1071 +#define _POP_TOP_FLOAT_r32 1072 +#define _POP_TOP_INT_r10 1073 +#define _POP_TOP_INT_r21 1074 +#define _POP_TOP_INT_r32 1075 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1076 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1077 +#define _POP_TOP_NOP_r10 1078 +#define _POP_TOP_NOP_r21 1079 +#define _POP_TOP_NOP_r32 1080 +#define _POP_TOP_UNICODE_r10 1081 +#define _POP_TOP_UNICODE_r21 1082 +#define _POP_TOP_UNICODE_r32 1083 +#define _POP_TWO_r20 1084 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1085 +#define _PUSH_EXC_INFO_r01 1086 +#define _PUSH_EXC_INFO_r12 1087 +#define _PUSH_EXC_INFO_r23 1088 +#define _PUSH_FRAME_r10 1089 +#define _PUSH_NULL_r01 1090 +#define _PUSH_NULL_r12 1091 +#define _PUSH_NULL_r23 1092 +#define _PUSH_NULL_CONDITIONAL_r00 1093 +#define _PY_FRAME_GENERAL_r01 1094 +#define _PY_FRAME_KW_r11 1095 +#define _QUICKEN_RESUME_r00 1096 +#define _QUICKEN_RESUME_r11 1097 +#define _QUICKEN_RESUME_r22 1098 +#define _QUICKEN_RESUME_r33 1099 +#define _REPLACE_WITH_TRUE_r11 1100 +#define _RESUME_CHECK_r00 1101 +#define _RESUME_CHECK_r11 1102 +#define _RESUME_CHECK_r22 1103 +#define _RESUME_CHECK_r33 1104 +#define _RETURN_GENERATOR_r01 1105 +#define _RETURN_VALUE_r11 1106 +#define _SAVE_RETURN_OFFSET_r00 1107 +#define _SAVE_RETURN_OFFSET_r11 1108 +#define _SAVE_RETURN_OFFSET_r22 1109 +#define _SAVE_RETURN_OFFSET_r33 1110 +#define _SEND_r22 1111 +#define _SEND_GEN_FRAME_r22 1112 +#define _SETUP_ANNOTATIONS_r00 1113 +#define _SET_ADD_r10 1114 +#define _SET_FUNCTION_ATTRIBUTE_r21 1115 +#define _SET_IP_r00 1116 +#define _SET_IP_r11 1117 +#define _SET_IP_r22 1118 +#define _SET_IP_r33 1119 +#define _SET_UPDATE_r10 1120 +#define _SPILL_OR_RELOAD_r01 1121 +#define _SPILL_OR_RELOAD_r02 1122 +#define _SPILL_OR_RELOAD_r03 1123 +#define _SPILL_OR_RELOAD_r10 1124 +#define _SPILL_OR_RELOAD_r12 1125 +#define _SPILL_OR_RELOAD_r13 1126 +#define _SPILL_OR_RELOAD_r20 1127 +#define _SPILL_OR_RELOAD_r21 1128 +#define _SPILL_OR_RELOAD_r23 1129 +#define _SPILL_OR_RELOAD_r30 1130 +#define _SPILL_OR_RELOAD_r31 1131 +#define _SPILL_OR_RELOAD_r32 1132 +#define _START_EXECUTOR_r00 1133 +#define _STORE_ATTR_r20 1134 +#define _STORE_ATTR_INSTANCE_VALUE_r20 1135 +#define _STORE_ATTR_SLOT_r20 1136 +#define _STORE_ATTR_WITH_HINT_r20 1137 +#define _STORE_DEREF_r10 1138 +#define _STORE_FAST_r10 1139 +#define _STORE_FAST_0_r10 1140 +#define _STORE_FAST_1_r10 1141 +#define _STORE_FAST_2_r10 1142 +#define _STORE_FAST_3_r10 1143 +#define _STORE_FAST_4_r10 1144 +#define _STORE_FAST_5_r10 1145 +#define _STORE_FAST_6_r10 1146 +#define _STORE_FAST_7_r10 1147 +#define _STORE_FAST_LOAD_FAST_r11 1148 +#define _STORE_FAST_STORE_FAST_r20 1149 +#define _STORE_GLOBAL_r10 1150 +#define _STORE_NAME_r10 1151 +#define _STORE_SLICE_r30 1152 +#define _STORE_SUBSCR_r30 1153 +#define _STORE_SUBSCR_DICT_r30 1154 +#define _STORE_SUBSCR_LIST_INT_r30 1155 +#define _SWAP_r11 1156 +#define _SWAP_2_r00 1157 +#define _SWAP_2_r11 1158 +#define _SWAP_2_r22 1159 +#define _SWAP_2_r33 1160 +#define _SWAP_3_r33 1161 +#define _TIER2_RESUME_CHECK_r00 1162 +#define _TIER2_RESUME_CHECK_r11 1163 +#define _TIER2_RESUME_CHECK_r22 1164 +#define _TIER2_RESUME_CHECK_r33 1165 +#define _TO_BOOL_r11 1166 +#define _TO_BOOL_BOOL_r00 1167 +#define _TO_BOOL_BOOL_r11 1168 +#define _TO_BOOL_BOOL_r22 1169 +#define _TO_BOOL_BOOL_r33 1170 +#define _TO_BOOL_INT_r11 1171 +#define _TO_BOOL_LIST_r11 1172 +#define _TO_BOOL_NONE_r00 1173 +#define _TO_BOOL_NONE_r11 1174 +#define _TO_BOOL_NONE_r22 1175 +#define _TO_BOOL_NONE_r33 1176 +#define _TO_BOOL_STR_r11 1177 +#define _UNARY_INVERT_r11 1178 +#define _UNARY_NEGATIVE_r11 1179 +#define _UNARY_NOT_r00 1180 +#define _UNARY_NOT_r11 1181 +#define _UNARY_NOT_r22 1182 +#define _UNARY_NOT_r33 1183 +#define _UNPACK_EX_r10 1184 +#define _UNPACK_SEQUENCE_r10 1185 +#define _UNPACK_SEQUENCE_LIST_r10 1186 +#define _UNPACK_SEQUENCE_TUPLE_r10 1187 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1188 +#define _WITH_EXCEPT_START_r33 1189 +#define _YIELD_VALUE_r11 1190 +#define MAX_UOP_REGS_ID 1190 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index bf361233560c55..0e769db65f66d2 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -11,13 +11,30 @@ extern "C" { #include #include "pycore_uop_ids.h" +#define MAX_CACHED_REGISTER 3 extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange; extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1]; -extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; +extern const char * const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1]; extern int _PyUop_num_popped(int opcode, int oparg); +typedef struct _pyuop_tos_cache_entry { + /* input depth is implicit in position */ + int8_t output; + int8_t exit; + uint16_t opcode; +} _PyUopTOSentry; + +typedef struct _PyUopCachingInfo { + uint8_t best[MAX_CACHED_REGISTER + 1]; + _PyUopTOSentry entries[MAX_CACHED_REGISTER + 1]; +} _PyUopCachingInfo; + +extern const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1]; +extern const uint16_t _PyUop_SpillsAndReloads[4][4]; +extern const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1]; + #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, @@ -44,8 +61,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_FAST_BORROW_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG, [_LOAD_SMALL_INT_0] = 0, [_LOAD_SMALL_INT_1] = 0, @@ -61,8 +76,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_POP_TOP_NOP] = 0, [_POP_TOP_INT] = 0, @@ -237,7 +250,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_FUNCTION_VERSION_INLINE] = HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -256,7 +269,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = 0, + [_PUSH_FRAME] = HAS_SYNC_SP_FLAG, [_GUARD_NOS_NULL] = HAS_DEOPT_FLAG, [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_GUARD_THIRD_NULL] = HAS_DEOPT_FLAG, @@ -267,7 +280,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_CALLABLE_TUPLE_1] = HAS_DEOPT_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -284,7 +297,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, @@ -330,14 +343,15 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_CONST_UNDER_INLINE] = 0, [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, - [_START_EXECUTOR] = HAS_DEOPT_FLAG, + [_START_EXECUTOR] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, - [_DEOPT] = 0, - [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, - [_ERROR_POP_N] = HAS_ARG_FLAG, + [_DEOPT] = HAS_SYNC_SP_FLAG, + [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, + [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_SYNC_SP_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, - [_COLD_EXIT] = HAS_ESCAPES_FLAG, + [_SPILL_OR_RELOAD] = 0, + [_COLD_EXIT] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { @@ -350,325 +364,4319 @@ const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { [_SWAP] = { 2, 4 }, }; -const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { +const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { + [_NOP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _NOP_r00 }, + { 1, 1, _NOP_r11 }, + { 2, 2, _NOP_r22 }, + { 3, 3, _NOP_r33 }, + }, + }, + [_CHECK_PERIODIC] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_PERIODIC_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_RESUME_CHECK] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _RESUME_CHECK_r00 }, + { 1, 1, _RESUME_CHECK_r11 }, + { 2, 2, _RESUME_CHECK_r22 }, + { 3, 3, _RESUME_CHECK_r33 }, + }, + }, + [_LOAD_FAST_CHECK] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_FAST_CHECK_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_0_r01 }, + { 2, 1, _LOAD_FAST_0_r12 }, + { 3, 2, _LOAD_FAST_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_1_r01 }, + { 2, 1, _LOAD_FAST_1_r12 }, + { 3, 2, _LOAD_FAST_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_2_r01 }, + { 2, 1, _LOAD_FAST_2_r12 }, + { 3, 2, _LOAD_FAST_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_3_r01 }, + { 2, 1, _LOAD_FAST_3_r12 }, + { 3, 2, _LOAD_FAST_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_4] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_4_r01 }, + { 2, 1, _LOAD_FAST_4_r12 }, + { 3, 2, _LOAD_FAST_4_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_5] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_5_r01 }, + { 2, 1, _LOAD_FAST_5_r12 }, + { 3, 2, _LOAD_FAST_5_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_6] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_6_r01 }, + { 2, 1, _LOAD_FAST_6_r12 }, + { 3, 2, _LOAD_FAST_6_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_7] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_7_r01 }, + { 2, 1, _LOAD_FAST_7_r12 }, + { 3, 2, _LOAD_FAST_7_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_r01 }, + { 2, 1, _LOAD_FAST_r12 }, + { 3, 2, _LOAD_FAST_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_0_r01 }, + { 2, 1, _LOAD_FAST_BORROW_0_r12 }, + { 3, 2, _LOAD_FAST_BORROW_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_1_r01 }, + { 2, 1, _LOAD_FAST_BORROW_1_r12 }, + { 3, 2, _LOAD_FAST_BORROW_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_2_r01 }, + { 2, 1, _LOAD_FAST_BORROW_2_r12 }, + { 3, 2, _LOAD_FAST_BORROW_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_3_r01 }, + { 2, 1, _LOAD_FAST_BORROW_3_r12 }, + { 3, 2, _LOAD_FAST_BORROW_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_4] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_4_r01 }, + { 2, 1, _LOAD_FAST_BORROW_4_r12 }, + { 3, 2, _LOAD_FAST_BORROW_4_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_5] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_5_r01 }, + { 2, 1, _LOAD_FAST_BORROW_5_r12 }, + { 3, 2, _LOAD_FAST_BORROW_5_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_6] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_6_r01 }, + { 2, 1, _LOAD_FAST_BORROW_6_r12 }, + { 3, 2, _LOAD_FAST_BORROW_6_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_7] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_7_r01 }, + { 2, 1, _LOAD_FAST_BORROW_7_r12 }, + { 3, 2, _LOAD_FAST_BORROW_7_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_r01 }, + { 2, 1, _LOAD_FAST_BORROW_r12 }, + { 3, 2, _LOAD_FAST_BORROW_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_AND_CLEAR] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_AND_CLEAR_r01 }, + { 2, 1, _LOAD_FAST_AND_CLEAR_r12 }, + { 3, 2, _LOAD_FAST_AND_CLEAR_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_CONST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_r01 }, + { 2, 1, _LOAD_CONST_r12 }, + { 3, 2, _LOAD_CONST_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_0_r01 }, + { 2, 1, _LOAD_SMALL_INT_0_r12 }, + { 3, 2, _LOAD_SMALL_INT_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_1_r01 }, + { 2, 1, _LOAD_SMALL_INT_1_r12 }, + { 3, 2, _LOAD_SMALL_INT_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_2_r01 }, + { 2, 1, _LOAD_SMALL_INT_2_r12 }, + { 3, 2, _LOAD_SMALL_INT_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_3_r01 }, + { 2, 1, _LOAD_SMALL_INT_3_r12 }, + { 3, 2, _LOAD_SMALL_INT_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_r01 }, + { 2, 1, _LOAD_SMALL_INT_r12 }, + { 3, 2, _LOAD_SMALL_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_STORE_FAST_0] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_FAST_0_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_FAST_1] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_FAST_1_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_FAST_2] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_FAST_2_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_FAST_3] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_FAST_3_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_FAST_4] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_FAST_4_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_FAST_5] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_FAST_5_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_FAST_6] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_FAST_6_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_FAST_7] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_FAST_7_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_FAST] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_FAST_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_TOP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_TOP_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_TOP_NOP] = { + .best = { 1, 1, 2, 3 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_TOP_NOP_r10 }, + { 1, 2, _POP_TOP_NOP_r21 }, + { 2, 3, _POP_TOP_NOP_r32 }, + }, + }, + [_POP_TOP_INT] = { + .best = { 1, 1, 2, 3 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_TOP_INT_r10 }, + { 1, 2, _POP_TOP_INT_r21 }, + { 2, 3, _POP_TOP_INT_r32 }, + }, + }, + [_POP_TOP_FLOAT] = { + .best = { 1, 1, 2, 3 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_TOP_FLOAT_r10 }, + { 1, 2, _POP_TOP_FLOAT_r21 }, + { 2, 3, _POP_TOP_FLOAT_r32 }, + }, + }, + [_POP_TOP_UNICODE] = { + .best = { 1, 1, 2, 3 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_TOP_UNICODE_r10 }, + { 1, 2, _POP_TOP_UNICODE_r21 }, + { 2, 3, _POP_TOP_UNICODE_r32 }, + }, + }, + [_POP_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _POP_TWO_r20 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_NULL] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _PUSH_NULL_r01 }, + { 2, 1, _PUSH_NULL_r12 }, + { 3, 2, _PUSH_NULL_r23 }, + { -1, -1, -1 }, + }, + }, + [_END_FOR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _END_FOR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_ITER] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _POP_ITER_r20 }, + { -1, -1, -1 }, + }, + }, + [_END_SEND] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _END_SEND_r21 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_NEGATIVE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _UNARY_NEGATIVE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_NOT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _UNARY_NOT_r00 }, + { 1, 1, _UNARY_NOT_r11 }, + { 2, 2, _UNARY_NOT_r22 }, + { 3, 3, _UNARY_NOT_r33 }, + }, + }, + [_TO_BOOL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _TO_BOOL_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_TO_BOOL_BOOL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _TO_BOOL_BOOL_r00 }, + { 1, 1, _TO_BOOL_BOOL_r11 }, + { 2, 2, _TO_BOOL_BOOL_r22 }, + { 3, 3, _TO_BOOL_BOOL_r33 }, + }, + }, + [_TO_BOOL_INT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _TO_BOOL_INT_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOS_LIST_r00 }, + { 1, 1, _GUARD_NOS_LIST_r11 }, + { 2, 2, _GUARD_NOS_LIST_r22 }, + { 3, 3, _GUARD_NOS_LIST_r33 }, + }, + }, + [_GUARD_TOS_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TOS_LIST_r00 }, + { 1, 1, _GUARD_TOS_LIST_r11 }, + { 2, 2, _GUARD_TOS_LIST_r22 }, + { 3, 3, _GUARD_TOS_LIST_r33 }, + }, + }, + [_GUARD_TOS_SLICE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TOS_SLICE_r00 }, + { 1, 1, _GUARD_TOS_SLICE_r11 }, + { 2, 2, _GUARD_TOS_SLICE_r22 }, + { 3, 3, _GUARD_TOS_SLICE_r33 }, + }, + }, + [_TO_BOOL_LIST] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _TO_BOOL_LIST_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_TO_BOOL_NONE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _TO_BOOL_NONE_r00 }, + { 1, 1, _TO_BOOL_NONE_r11 }, + { 2, 2, _TO_BOOL_NONE_r22 }, + { 3, 3, _TO_BOOL_NONE_r33 }, + }, + }, + [_GUARD_NOS_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOS_UNICODE_r00 }, + { 1, 1, _GUARD_NOS_UNICODE_r11 }, + { 2, 2, _GUARD_NOS_UNICODE_r22 }, + { 3, 3, _GUARD_NOS_UNICODE_r33 }, + }, + }, + [_GUARD_TOS_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TOS_UNICODE_r00 }, + { 1, 1, _GUARD_TOS_UNICODE_r11 }, + { 2, 2, _GUARD_TOS_UNICODE_r22 }, + { 3, 3, _GUARD_TOS_UNICODE_r33 }, + }, + }, + [_TO_BOOL_STR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _TO_BOOL_STR_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_REPLACE_WITH_TRUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _REPLACE_WITH_TRUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_INVERT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _UNARY_INVERT_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOS_INT_r00 }, + { 1, 1, _GUARD_NOS_INT_r11 }, + { 2, 2, _GUARD_NOS_INT_r22 }, + { 3, 3, _GUARD_NOS_INT_r33 }, + }, + }, + [_GUARD_TOS_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TOS_INT_r00 }, + { 1, 1, _GUARD_TOS_INT_r11 }, + { 2, 2, _GUARD_TOS_INT_r22 }, + { 3, 3, _GUARD_TOS_INT_r33 }, + }, + }, + [_GUARD_NOS_OVERFLOWED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOS_OVERFLOWED_r00 }, + { 1, 1, _GUARD_NOS_OVERFLOWED_r11 }, + { 2, 2, _GUARD_NOS_OVERFLOWED_r22 }, + { 3, 3, _GUARD_NOS_OVERFLOWED_r33 }, + }, + }, + [_GUARD_TOS_OVERFLOWED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TOS_OVERFLOWED_r00 }, + { 1, 1, _GUARD_TOS_OVERFLOWED_r11 }, + { 2, 2, _GUARD_TOS_OVERFLOWED_r22 }, + { 3, 3, _GUARD_TOS_OVERFLOWED_r33 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_MULTIPLY_INT_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_ADD_INT_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_SUBTRACT_INT_r21 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOS_FLOAT_r00 }, + { 1, 1, _GUARD_NOS_FLOAT_r11 }, + { 2, 2, _GUARD_NOS_FLOAT_r22 }, + { 3, 3, _GUARD_NOS_FLOAT_r33 }, + }, + }, + [_GUARD_TOS_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TOS_FLOAT_r00 }, + { 1, 1, _GUARD_TOS_FLOAT_r11 }, + { 2, 2, _GUARD_TOS_FLOAT_r22 }, + { 3, 3, _GUARD_TOS_FLOAT_r33 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_MULTIPLY_FLOAT_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_ADD_FLOAT_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_SUBTRACT_FLOAT_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_UNICODE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_ADD_UNICODE_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_INPLACE_ADD_UNICODE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 0, _BINARY_OP_INPLACE_ADD_UNICODE_r20 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 0, _GUARD_BINARY_OP_EXTEND_r22 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_EXTEND] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _BINARY_OP_EXTEND_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_SLICE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _BINARY_SLICE_r31 }, + }, + }, + [_STORE_SLICE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 0, _STORE_SLICE_r30 }, + }, + }, + [_BINARY_OP_SUBSCR_LIST_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _BINARY_OP_SUBSCR_LIST_INT_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_LIST_SLICE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _BINARY_OP_SUBSCR_LIST_SLICE_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_STR_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_SUBSCR_STR_INT_r21 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOS_TUPLE_r00 }, + { 1, 1, _GUARD_NOS_TUPLE_r11 }, + { 2, 2, _GUARD_NOS_TUPLE_r22 }, + { 3, 3, _GUARD_NOS_TUPLE_r33 }, + }, + }, + [_GUARD_TOS_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TOS_TUPLE_r00 }, + { 1, 1, _GUARD_TOS_TUPLE_r11 }, + { 2, 2, _GUARD_TOS_TUPLE_r22 }, + { 3, 3, _GUARD_TOS_TUPLE_r33 }, + }, + }, + [_BINARY_OP_SUBSCR_TUPLE_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_SUBSCR_TUPLE_INT_r21 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOS_DICT_r00 }, + { 1, 1, _GUARD_NOS_DICT_r11 }, + { 2, 2, _GUARD_NOS_DICT_r22 }, + { 3, 3, _GUARD_NOS_DICT_r33 }, + }, + }, + [_GUARD_TOS_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TOS_DICT_r00 }, + { 1, 1, _GUARD_TOS_DICT_r11 }, + { 2, 2, _GUARD_TOS_DICT_r22 }, + { 3, 3, _GUARD_TOS_DICT_r33 }, + }, + }, + [_BINARY_OP_SUBSCR_DICT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _BINARY_OP_SUBSCR_DICT_r21 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_CHECK_FUNC] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_CHECK_FUNC_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_INIT_CALL] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _BINARY_OP_SUBSCR_INIT_CALL_r31 }, + }, + }, + [_LIST_APPEND] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _LIST_APPEND_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_ADD] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _SET_ADD_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_SUBSCR] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 0, _STORE_SUBSCR_r30 }, + }, + }, + [_STORE_SUBSCR_LIST_INT] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _STORE_SUBSCR_LIST_INT_r30 }, + }, + }, + [_STORE_SUBSCR_DICT] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 0, _STORE_SUBSCR_DICT_r30 }, + }, + }, + [_DELETE_SUBSCR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 0, _DELETE_SUBSCR_r20 }, + { -1, -1, -1 }, + }, + }, + [_CALL_INTRINSIC_1] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _CALL_INTRINSIC_1_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_INTRINSIC_2] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _CALL_INTRINSIC_2_r21 }, + { -1, -1, -1 }, + }, + }, + [_RETURN_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _RETURN_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_AITER] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _GET_AITER_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_ANEXT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 0, _GET_ANEXT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_AWAITABLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _GET_AWAITABLE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SEND_GEN_FRAME] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 2, _SEND_GEN_FRAME_r22 }, + { -1, -1, -1 }, + }, + }, + [_YIELD_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _YIELD_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_EXCEPT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _POP_EXCEPT_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_COMMON_CONSTANT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_COMMON_CONSTANT_r01 }, + { 2, 1, _LOAD_COMMON_CONSTANT_r12 }, + { 3, 2, _LOAD_COMMON_CONSTANT_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_BUILD_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_BUILD_CLASS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_NAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _STORE_NAME_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_NAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_NAME_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _UNPACK_SEQUENCE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_TWO_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNPACK_SEQUENCE_TWO_TUPLE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_TUPLE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_LIST] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_LIST_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_EX] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _UNPACK_EX_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_ATTR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 0, _STORE_ATTR_r20 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_ATTR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _DELETE_ATTR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_GLOBAL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _STORE_GLOBAL_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_GLOBAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_GLOBAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_LOCALS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_LOCALS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_NAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_NAME_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_GLOBAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _LOAD_GLOBAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_NULL_CONDITIONAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _PUSH_NULL_CONDITIONAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_GLOBALS_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_GLOBALS_VERSION_r00 }, + { 1, 1, _GUARD_GLOBALS_VERSION_r11 }, + { 2, 2, _GUARD_GLOBALS_VERSION_r22 }, + { 3, 3, _GUARD_GLOBALS_VERSION_r33 }, + }, + }, + [_LOAD_GLOBAL_MODULE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_GLOBAL_MODULE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_GLOBAL_BUILTINS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_GLOBAL_BUILTINS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_CELL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _MAKE_CELL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_DEREF] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_DEREF_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FROM_DICT_OR_DEREF] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _LOAD_FROM_DICT_OR_DEREF_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_DEREF] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_DEREF_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_DEREF] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _STORE_DEREF_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_COPY_FREE_VARS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _COPY_FREE_VARS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_STRING] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_STRING_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_INTERPOLATION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_INTERPOLATION_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_TEMPLATE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _BUILD_TEMPLATE_r21 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_TUPLE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_TUPLE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_LIST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_LIST_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LIST_EXTEND] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _LIST_EXTEND_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_UPDATE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _SET_UPDATE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_SET] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_SET_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_MAP] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_MAP_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SETUP_ANNOTATIONS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _SETUP_ANNOTATIONS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DICT_UPDATE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _DICT_UPDATE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DICT_MERGE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _DICT_MERGE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAP_ADD] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 0, _MAP_ADD_r20 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SUPER_ATTR_ATTR] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _LOAD_SUPER_ATTR_ATTR_r31 }, + }, + }, + [_LOAD_SUPER_ATTR_METHOD] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 0, _LOAD_SUPER_ATTR_METHOD_r32 }, + }, + }, + [_LOAD_ATTR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _LOAD_ATTR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_TYPE_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TYPE_VERSION_r00 }, + { 1, 1, _GUARD_TYPE_VERSION_r11 }, + { 2, 2, _GUARD_TYPE_VERSION_r22 }, + { 3, 3, _GUARD_TYPE_VERSION_r33 }, + }, + }, + [_GUARD_TYPE_VERSION_AND_LOCK] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TYPE_VERSION_AND_LOCK_r00 }, + { 1, 1, _GUARD_TYPE_VERSION_AND_LOCK_r11 }, + { 2, 2, _GUARD_TYPE_VERSION_AND_LOCK_r22 }, + { 3, 3, _GUARD_TYPE_VERSION_AND_LOCK_r33 }, + }, + }, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_MANAGED_OBJECT_HAS_VALUES_r00 }, + { 1, 1, _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 }, + { 2, 2, _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 }, + { 3, 3, _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 }, + }, + }, + [_LOAD_ATTR_INSTANCE_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_INSTANCE_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_MODULE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_MODULE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_WITH_HINT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_WITH_HINT_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_SLOT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_SLOT_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_ATTR_CLASS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_ATTR_CLASS_r00 }, + { 1, 1, _CHECK_ATTR_CLASS_r11 }, + { 2, 2, _CHECK_ATTR_CLASS_r22 }, + { 3, 3, _CHECK_ATTR_CLASS_r33 }, + }, + }, + [_LOAD_ATTR_CLASS] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_CLASS_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_PROPERTY_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_PROPERTY_FRAME_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_DORV_NO_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_DORV_NO_DICT_r00 }, + { 1, 1, _GUARD_DORV_NO_DICT_r11 }, + { 2, 2, _GUARD_DORV_NO_DICT_r22 }, + { 3, 3, _GUARD_DORV_NO_DICT_r33 }, + }, + }, + [_STORE_ATTR_INSTANCE_VALUE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _STORE_ATTR_INSTANCE_VALUE_r20 }, + { -1, -1, -1 }, + }, + }, + [_STORE_ATTR_WITH_HINT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 0, _STORE_ATTR_WITH_HINT_r20 }, + { -1, -1, -1 }, + }, + }, + [_STORE_ATTR_SLOT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _STORE_ATTR_SLOT_r20 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _COMPARE_OP_r21 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_FLOAT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _COMPARE_OP_FLOAT_r21 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _COMPARE_OP_INT_r21 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_STR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _COMPARE_OP_STR_r21 }, + { -1, -1, -1 }, + }, + }, + [_IS_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _IS_OP_r21 }, + { -1, -1, -1 }, + }, + }, + [_CONTAINS_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _CONTAINS_OP_r21 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_TOS_ANY_SET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_TOS_ANY_SET_r00 }, + { 1, 1, _GUARD_TOS_ANY_SET_r11 }, + { 2, 2, _GUARD_TOS_ANY_SET_r22 }, + { 3, 3, _GUARD_TOS_ANY_SET_r33 }, + }, + }, + [_CONTAINS_OP_SET] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _CONTAINS_OP_SET_r21 }, + { -1, -1, -1 }, + }, + }, + [_CONTAINS_OP_DICT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _CONTAINS_OP_DICT_r21 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_EG_MATCH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 0, _CHECK_EG_MATCH_r22 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_EXC_MATCH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 0, _CHECK_EXC_MATCH_r22 }, + { -1, -1, -1 }, + }, + }, + [_IMPORT_NAME] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _IMPORT_NAME_r21 }, + { -1, -1, -1 }, + }, + }, + [_IMPORT_FROM] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 0, _IMPORT_FROM_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_IS_NONE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _IS_NONE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_LEN] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 0, _GET_LEN_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_CLASS] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _MATCH_CLASS_r31 }, + }, + }, + [_MATCH_MAPPING] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _MATCH_MAPPING_r01 }, + { 2, 1, _MATCH_MAPPING_r12 }, + { 3, 2, _MATCH_MAPPING_r23 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_SEQUENCE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _MATCH_SEQUENCE_r01 }, + { 2, 1, _MATCH_SEQUENCE_r12 }, + { 3, 2, _MATCH_SEQUENCE_r23 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_KEYS] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 0, _MATCH_KEYS_r23 }, + { -1, -1, -1 }, + }, + }, + [_GET_ITER] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 0, _GET_ITER_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_YIELD_FROM_ITER] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _GET_YIELD_FROM_ITER_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FOR_ITER_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 0, _FOR_ITER_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _ITER_CHECK_LIST_r00 }, + { 1, 1, _ITER_CHECK_LIST_r11 }, + { 2, 2, _ITER_CHECK_LIST_r22 }, + { 3, 3, _ITER_CHECK_LIST_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOT_EXHAUSTED_LIST_r00 }, + { 1, 1, _GUARD_NOT_EXHAUSTED_LIST_r11 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_LIST_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_LIST_r33 }, + }, + }, + [_ITER_NEXT_LIST_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 0, _ITER_NEXT_LIST_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _ITER_CHECK_TUPLE_r00 }, + { 1, 1, _ITER_CHECK_TUPLE_r11 }, + { 2, 2, _ITER_CHECK_TUPLE_r22 }, + { 3, 3, _ITER_CHECK_TUPLE_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOT_EXHAUSTED_TUPLE_r00 }, + { 1, 1, _GUARD_NOT_EXHAUSTED_TUPLE_r11 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_TUPLE_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_TUPLE_r33 }, + }, + }, + [_ITER_NEXT_TUPLE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _ITER_NEXT_TUPLE_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_RANGE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _ITER_CHECK_RANGE_r00 }, + { 1, 1, _ITER_CHECK_RANGE_r11 }, + { 2, 2, _ITER_CHECK_RANGE_r22 }, + { 3, 3, _ITER_CHECK_RANGE_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_RANGE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOT_EXHAUSTED_RANGE_r00 }, + { 1, 1, _GUARD_NOT_EXHAUSTED_RANGE_r11 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_RANGE_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_RANGE_r33 }, + }, + }, + [_ITER_NEXT_RANGE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _ITER_NEXT_RANGE_r23 }, + { -1, -1, -1 }, + }, + }, + [_FOR_ITER_GEN_FRAME] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _FOR_ITER_GEN_FRAME_r23 }, + { -1, -1, -1 }, + }, + }, + [_INSERT_NULL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _INSERT_NULL_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SPECIAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _LOAD_SPECIAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_WITH_EXCEPT_START] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 0, _WITH_EXCEPT_START_r33 }, + }, + }, + [_PUSH_EXC_INFO] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _PUSH_EXC_INFO_r01 }, + { 2, 1, _PUSH_EXC_INFO_r12 }, + { 3, 2, _PUSH_EXC_INFO_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00 }, + { 1, 1, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 }, + { 2, 2, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 }, + { 3, 3, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 }, + }, + }, + [_GUARD_KEYS_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_KEYS_VERSION_r00 }, + { 1, 1, _GUARD_KEYS_VERSION_r11 }, + { 2, 2, _GUARD_KEYS_VERSION_r22 }, + { 3, 3, _GUARD_KEYS_VERSION_r33 }, + }, + }, + [_LOAD_ATTR_METHOD_WITH_VALUES] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_ATTR_METHOD_WITH_VALUES_r01 }, + { 2, 1, _LOAD_ATTR_METHOD_WITH_VALUES_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_WITH_VALUES_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_METHOD_NO_DICT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_ATTR_METHOD_NO_DICT_r01 }, + { 2, 1, _LOAD_ATTR_METHOD_NO_DICT_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_NO_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_ATTR_METHOD_LAZY_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_ATTR_METHOD_LAZY_DICT_r00 }, + { 1, 1, _CHECK_ATTR_METHOD_LAZY_DICT_r11 }, + { 2, 2, _CHECK_ATTR_METHOD_LAZY_DICT_r22 }, + { 3, 3, _CHECK_ATTR_METHOD_LAZY_DICT_r33 }, + }, + }, + [_LOAD_ATTR_METHOD_LAZY_DICT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_ATTR_METHOD_LAZY_DICT_r01 }, + { 2, 1, _LOAD_ATTR_METHOD_LAZY_DICT_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_LAZY_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_MAYBE_EXPAND_METHOD] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _MAYBE_EXPAND_METHOD_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PY_FRAME_GENERAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _PY_FRAME_GENERAL_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_VERSION_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION_INLINE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_VERSION_INLINE_r00 }, + { 1, 1, _CHECK_FUNCTION_VERSION_INLINE_r11 }, + { 2, 2, _CHECK_FUNCTION_VERSION_INLINE_r22 }, + { 3, 3, _CHECK_FUNCTION_VERSION_INLINE_r33 }, + }, + }, + [_CHECK_METHOD_VERSION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_METHOD_VERSION_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXPAND_METHOD] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _EXPAND_METHOD_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_IS_NOT_PY_CALLABLE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_NON_PY_GENERAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_NON_PY_GENERAL_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_PEP_523] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_PEP_523_r00 }, + { 1, 1, _CHECK_PEP_523_r11 }, + { 2, 2, _CHECK_PEP_523_r22 }, + { 3, 3, _CHECK_PEP_523_r33 }, + }, + }, + [_CHECK_FUNCTION_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_STACK_SPACE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_STACK_SPACE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_RECURSION_REMAINING] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_RECURSION_REMAINING_r00 }, + { 1, 1, _CHECK_RECURSION_REMAINING_r11 }, + { 2, 2, _CHECK_RECURSION_REMAINING_r22 }, + { 3, 3, _CHECK_RECURSION_REMAINING_r33 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_0] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_0_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_1] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_1_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_2] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_2_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_3] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_3_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_4] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_4_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _PUSH_FRAME_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOS_NULL_r00 }, + { 1, 1, _GUARD_NOS_NULL_r11 }, + { 2, 2, _GUARD_NOS_NULL_r22 }, + { 3, 3, _GUARD_NOS_NULL_r33 }, + }, + }, + [_GUARD_NOS_NOT_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOS_NOT_NULL_r00 }, + { 1, 1, _GUARD_NOS_NOT_NULL_r11 }, + { 2, 2, _GUARD_NOS_NOT_NULL_r22 }, + { 3, 3, _GUARD_NOS_NOT_NULL_r33 }, + }, + }, + [_GUARD_THIRD_NULL] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _GUARD_THIRD_NULL_r33 }, + }, + }, + [_GUARD_CALLABLE_TYPE_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _GUARD_CALLABLE_TYPE_1_r33 }, + }, + }, + [_CALL_TYPE_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _CALL_TYPE_1_r31 }, + }, + }, + [_GUARD_CALLABLE_STR_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _GUARD_CALLABLE_STR_1_r33 }, + }, + }, + [_CALL_STR_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _CALL_STR_1_r31 }, + }, + }, + [_GUARD_CALLABLE_TUPLE_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _GUARD_CALLABLE_TUPLE_1_r33 }, + }, + }, + [_CALL_TUPLE_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _CALL_TUPLE_1_r31 }, + }, + }, + [_CHECK_AND_ALLOCATE_OBJECT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_AND_ALLOCATE_OBJECT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CREATE_INIT_FRAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CREATE_INIT_FRAME_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXIT_INIT_CHECK] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _EXIT_INIT_CHECK_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_BUILTIN_CLASS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_BUILTIN_O_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_BUILTIN_FAST_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_LEN] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _GUARD_CALLABLE_LEN_r33 }, + }, + }, + [_CALL_LEN] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _CALL_LEN_r31 }, + }, + }, + [_GUARD_CALLABLE_ISINSTANCE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _GUARD_CALLABLE_ISINSTANCE_r33 }, + }, + }, + [_CALL_ISINSTANCE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _CALL_ISINSTANCE_r31 }, + }, + }, + [_GUARD_CALLABLE_LIST_APPEND] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _GUARD_CALLABLE_LIST_APPEND_r33 }, + }, + }, + [_CALL_LIST_APPEND] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _CALL_LIST_APPEND_r30 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_METHOD_DESCRIPTOR_O_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_METHOD_DESCRIPTOR_NOARGS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAYBE_EXPAND_METHOD_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _MAYBE_EXPAND_METHOD_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PY_FRAME_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _PY_FRAME_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_FUNCTION_VERSION_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_METHOD_VERSION_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_METHOD_VERSION_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXPAND_METHOD_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _EXPAND_METHOD_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_IS_NOT_PY_CALLABLE_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_KW_NON_PY] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _CALL_KW_NON_PY_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_CALLARGS_A_TUPLE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 0, _MAKE_CALLARGS_A_TUPLE_r33 }, + }, + }, + [_MAKE_FUNCTION] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _MAKE_FUNCTION_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_FUNCTION_ATTRIBUTE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _SET_FUNCTION_ATTRIBUTE_r21 }, + { -1, -1, -1 }, + }, + }, + [_RETURN_GENERATOR] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _RETURN_GENERATOR_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_SLICE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_SLICE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CONVERT_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _CONVERT_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FORMAT_SIMPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 0, _FORMAT_SIMPLE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FORMAT_WITH_SPEC] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _FORMAT_WITH_SPEC_r21 }, + { -1, -1, -1 }, + }, + }, + [_COPY_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _COPY_1_r01 }, + { 2, 1, _COPY_1_r12 }, + { 3, 2, _COPY_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_COPY_2] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _COPY_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_COPY_3] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _COPY_3_r33 }, + }, + }, + [_COPY] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _COPY_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 0, _BINARY_OP_r21 }, + { -1, -1, -1 }, + }, + }, + [_SWAP_2] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _SWAP_2_r00 }, + { 1, 1, _SWAP_2_r11 }, + { 2, 2, _SWAP_2_r22 }, + { 3, 3, _SWAP_2_r33 }, + }, + }, + [_SWAP_3] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _SWAP_3_r33 }, + }, + }, + [_SWAP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _SWAP_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_IS_TRUE_POP] = { + .best = { 1, 1, 2, 3 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _GUARD_IS_TRUE_POP_r10 }, + { 1, 1, _GUARD_IS_TRUE_POP_r21 }, + { 2, 2, _GUARD_IS_TRUE_POP_r32 }, + }, + }, + [_GUARD_IS_FALSE_POP] = { + .best = { 1, 1, 2, 3 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _GUARD_IS_FALSE_POP_r10 }, + { 1, 1, _GUARD_IS_FALSE_POP_r21 }, + { 2, 2, _GUARD_IS_FALSE_POP_r32 }, + }, + }, + [_GUARD_IS_NONE_POP] = { + .best = { 1, 1, 2, 3 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _GUARD_IS_NONE_POP_r10 }, + { 1, 1, _GUARD_IS_NONE_POP_r21 }, + { 2, 2, _GUARD_IS_NONE_POP_r32 }, + }, + }, + [_GUARD_IS_NOT_NONE_POP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _GUARD_IS_NOT_NONE_POP_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_JUMP_TO_TOP] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _JUMP_TO_TOP_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_IP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _SET_IP_r00 }, + { 1, 1, _SET_IP_r11 }, + { 2, 2, _SET_IP_r22 }, + { 3, 3, _SET_IP_r33 }, + }, + }, + [_CHECK_STACK_SPACE_OPERAND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_STACK_SPACE_OPERAND_r00 }, + { 1, 1, _CHECK_STACK_SPACE_OPERAND_r11 }, + { 2, 2, _CHECK_STACK_SPACE_OPERAND_r22 }, + { 3, 3, _CHECK_STACK_SPACE_OPERAND_r33 }, + }, + }, + [_SAVE_RETURN_OFFSET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _SAVE_RETURN_OFFSET_r00 }, + { 1, 1, _SAVE_RETURN_OFFSET_r11 }, + { 2, 2, _SAVE_RETURN_OFFSET_r22 }, + { 3, 3, _SAVE_RETURN_OFFSET_r33 }, + }, + }, + [_EXIT_TRACE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _EXIT_TRACE_r00 }, + { 0, 0, _EXIT_TRACE_r10 }, + { 0, 0, _EXIT_TRACE_r20 }, + { 0, 0, _EXIT_TRACE_r30 }, + }, + }, + [_CHECK_VALIDITY] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_VALIDITY_r00 }, + { 1, 1, _CHECK_VALIDITY_r11 }, + { 2, 2, _CHECK_VALIDITY_r22 }, + { 3, 3, _CHECK_VALIDITY_r33 }, + }, + }, + [_LOAD_CONST_INLINE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_INLINE_r01 }, + { 2, 1, _LOAD_CONST_INLINE_r12 }, + { 3, 2, _LOAD_CONST_INLINE_r23 }, + { -1, -1, -1 }, + }, + }, + [_POP_TOP_LOAD_CONST_INLINE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _POP_TOP_LOAD_CONST_INLINE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_CONST_INLINE_BORROW] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_INLINE_BORROW_r01 }, + { 2, 1, _LOAD_CONST_INLINE_BORROW_r12 }, + { 3, 2, _LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + }, + }, + [_POP_CALL] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _POP_CALL_r20 }, + { -1, -1, -1 }, + }, + }, + [_POP_CALL_ONE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _POP_CALL_ONE_r30 }, + }, + }, + [_POP_CALL_TWO] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _POP_CALL_TWO_r30 }, + }, + }, + [_POP_TOP_LOAD_CONST_INLINE_BORROW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_TWO_LOAD_CONST_INLINE_BORROW] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 }, + { -1, -1, -1 }, + }, + }, + [_POP_CALL_LOAD_CONST_INLINE_BORROW] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 }, + { -1, -1, -1 }, + }, + }, + [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 }, + }, + }, + [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 }, + }, + }, + [_LOAD_CONST_UNDER_INLINE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_UNDER_INLINE_r01 }, + { 2, 1, _LOAD_CONST_UNDER_INLINE_r12 }, + { 3, 2, _LOAD_CONST_UNDER_INLINE_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_CONST_UNDER_INLINE_BORROW] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_UNDER_INLINE_BORROW_r01 }, + { 2, 1, _LOAD_CONST_UNDER_INLINE_BORROW_r12 }, + { 3, 2, _LOAD_CONST_UNDER_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_r00 }, + { 1, 1, _CHECK_FUNCTION_r11 }, + { 2, 2, _CHECK_FUNCTION_r22 }, + { 3, 3, _CHECK_FUNCTION_r33 }, + }, + }, + [_START_EXECUTOR] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _START_EXECUTOR_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_WARM] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _MAKE_WARM_r00 }, + { 1, 1, _MAKE_WARM_r11 }, + { 2, 2, _MAKE_WARM_r22 }, + { 3, 3, _MAKE_WARM_r33 }, + }, + }, + [_FATAL_ERROR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _FATAL_ERROR_r00 }, + { 1, 1, _FATAL_ERROR_r11 }, + { 2, 2, _FATAL_ERROR_r22 }, + { 3, 3, _FATAL_ERROR_r33 }, + }, + }, + [_DEOPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _DEOPT_r00 }, + { 0, 0, _DEOPT_r10 }, + { 0, 0, _DEOPT_r20 }, + { 0, 0, _DEOPT_r30 }, + }, + }, + [_HANDLE_PENDING_AND_DEOPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r00 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r10 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r20 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r30 }, + }, + }, + [_ERROR_POP_N] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _ERROR_POP_N_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_TIER2_RESUME_CHECK] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _TIER2_RESUME_CHECK_r00 }, + { 1, 1, _TIER2_RESUME_CHECK_r11 }, + { 2, 2, _TIER2_RESUME_CHECK_r22 }, + { 3, 3, _TIER2_RESUME_CHECK_r33 }, + }, + }, + [_COLD_EXIT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _COLD_EXIT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, +}; + +const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { + [_NOP_r00] = _NOP, + [_NOP_r11] = _NOP, + [_NOP_r22] = _NOP, + [_NOP_r33] = _NOP, + [_CHECK_PERIODIC_r00] = _CHECK_PERIODIC, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = _CHECK_PERIODIC_IF_NOT_YIELD_FROM, + [_RESUME_CHECK_r00] = _RESUME_CHECK, + [_RESUME_CHECK_r11] = _RESUME_CHECK, + [_RESUME_CHECK_r22] = _RESUME_CHECK, + [_RESUME_CHECK_r33] = _RESUME_CHECK, + [_LOAD_FAST_CHECK_r01] = _LOAD_FAST_CHECK, + [_LOAD_FAST_0_r01] = _LOAD_FAST_0, + [_LOAD_FAST_0_r12] = _LOAD_FAST_0, + [_LOAD_FAST_0_r23] = _LOAD_FAST_0, + [_LOAD_FAST_1_r01] = _LOAD_FAST_1, + [_LOAD_FAST_1_r12] = _LOAD_FAST_1, + [_LOAD_FAST_1_r23] = _LOAD_FAST_1, + [_LOAD_FAST_2_r01] = _LOAD_FAST_2, + [_LOAD_FAST_2_r12] = _LOAD_FAST_2, + [_LOAD_FAST_2_r23] = _LOAD_FAST_2, + [_LOAD_FAST_3_r01] = _LOAD_FAST_3, + [_LOAD_FAST_3_r12] = _LOAD_FAST_3, + [_LOAD_FAST_3_r23] = _LOAD_FAST_3, + [_LOAD_FAST_4_r01] = _LOAD_FAST_4, + [_LOAD_FAST_4_r12] = _LOAD_FAST_4, + [_LOAD_FAST_4_r23] = _LOAD_FAST_4, + [_LOAD_FAST_5_r01] = _LOAD_FAST_5, + [_LOAD_FAST_5_r12] = _LOAD_FAST_5, + [_LOAD_FAST_5_r23] = _LOAD_FAST_5, + [_LOAD_FAST_6_r01] = _LOAD_FAST_6, + [_LOAD_FAST_6_r12] = _LOAD_FAST_6, + [_LOAD_FAST_6_r23] = _LOAD_FAST_6, + [_LOAD_FAST_7_r01] = _LOAD_FAST_7, + [_LOAD_FAST_7_r12] = _LOAD_FAST_7, + [_LOAD_FAST_7_r23] = _LOAD_FAST_7, + [_LOAD_FAST_r01] = _LOAD_FAST, + [_LOAD_FAST_r12] = _LOAD_FAST, + [_LOAD_FAST_r23] = _LOAD_FAST, + [_LOAD_FAST_BORROW_0_r01] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_0_r12] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_0_r23] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_1_r01] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_1_r12] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_1_r23] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_2_r01] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_2_r12] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_2_r23] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_3_r01] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_3_r12] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_3_r23] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_4_r01] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_4_r12] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_4_r23] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_5_r01] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_5_r12] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_5_r23] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_6_r01] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_6_r12] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_6_r23] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_7_r01] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_7_r12] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_7_r23] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_r01] = _LOAD_FAST_BORROW, + [_LOAD_FAST_BORROW_r12] = _LOAD_FAST_BORROW, + [_LOAD_FAST_BORROW_r23] = _LOAD_FAST_BORROW, + [_LOAD_FAST_AND_CLEAR_r01] = _LOAD_FAST_AND_CLEAR, + [_LOAD_FAST_AND_CLEAR_r12] = _LOAD_FAST_AND_CLEAR, + [_LOAD_FAST_AND_CLEAR_r23] = _LOAD_FAST_AND_CLEAR, + [_LOAD_CONST_r01] = _LOAD_CONST, + [_LOAD_CONST_r12] = _LOAD_CONST, + [_LOAD_CONST_r23] = _LOAD_CONST, + [_LOAD_SMALL_INT_0_r01] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_0_r12] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_0_r23] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_1_r01] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_1_r12] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_1_r23] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_2_r01] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_2_r12] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_2_r23] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_3_r01] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_3_r12] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_3_r23] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_r01] = _LOAD_SMALL_INT, + [_LOAD_SMALL_INT_r12] = _LOAD_SMALL_INT, + [_LOAD_SMALL_INT_r23] = _LOAD_SMALL_INT, + [_STORE_FAST_0_r10] = _STORE_FAST_0, + [_STORE_FAST_1_r10] = _STORE_FAST_1, + [_STORE_FAST_2_r10] = _STORE_FAST_2, + [_STORE_FAST_3_r10] = _STORE_FAST_3, + [_STORE_FAST_4_r10] = _STORE_FAST_4, + [_STORE_FAST_5_r10] = _STORE_FAST_5, + [_STORE_FAST_6_r10] = _STORE_FAST_6, + [_STORE_FAST_7_r10] = _STORE_FAST_7, + [_STORE_FAST_r10] = _STORE_FAST, + [_POP_TOP_r10] = _POP_TOP, + [_POP_TOP_NOP_r10] = _POP_TOP_NOP, + [_POP_TOP_NOP_r21] = _POP_TOP_NOP, + [_POP_TOP_NOP_r32] = _POP_TOP_NOP, + [_POP_TOP_INT_r10] = _POP_TOP_INT, + [_POP_TOP_INT_r21] = _POP_TOP_INT, + [_POP_TOP_INT_r32] = _POP_TOP_INT, + [_POP_TOP_FLOAT_r10] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r21] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r32] = _POP_TOP_FLOAT, + [_POP_TOP_UNICODE_r10] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r21] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r32] = _POP_TOP_UNICODE, + [_POP_TWO_r20] = _POP_TWO, + [_PUSH_NULL_r01] = _PUSH_NULL, + [_PUSH_NULL_r12] = _PUSH_NULL, + [_PUSH_NULL_r23] = _PUSH_NULL, + [_END_FOR_r10] = _END_FOR, + [_POP_ITER_r20] = _POP_ITER, + [_END_SEND_r21] = _END_SEND, + [_UNARY_NEGATIVE_r11] = _UNARY_NEGATIVE, + [_UNARY_NOT_r11] = _UNARY_NOT, + [_UNARY_NOT_r00] = _UNARY_NOT, + [_UNARY_NOT_r22] = _UNARY_NOT, + [_UNARY_NOT_r33] = _UNARY_NOT, + [_TO_BOOL_r11] = _TO_BOOL, + [_TO_BOOL_BOOL_r11] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r00] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r22] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r33] = _TO_BOOL_BOOL, + [_TO_BOOL_INT_r11] = _TO_BOOL_INT, + [_GUARD_NOS_LIST_r22] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r00] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r11] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r33] = _GUARD_NOS_LIST, + [_GUARD_TOS_LIST_r11] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r00] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r22] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r33] = _GUARD_TOS_LIST, + [_GUARD_TOS_SLICE_r11] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r00] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r22] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r33] = _GUARD_TOS_SLICE, + [_TO_BOOL_LIST_r11] = _TO_BOOL_LIST, + [_TO_BOOL_NONE_r11] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r00] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r22] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r33] = _TO_BOOL_NONE, + [_GUARD_NOS_UNICODE_r22] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r00] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r11] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r33] = _GUARD_NOS_UNICODE, + [_GUARD_TOS_UNICODE_r11] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r00] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r22] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r33] = _GUARD_TOS_UNICODE, + [_TO_BOOL_STR_r11] = _TO_BOOL_STR, + [_REPLACE_WITH_TRUE_r11] = _REPLACE_WITH_TRUE, + [_UNARY_INVERT_r11] = _UNARY_INVERT, + [_GUARD_NOS_INT_r22] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r00] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r11] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r33] = _GUARD_NOS_INT, + [_GUARD_TOS_INT_r11] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r00] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r22] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r33] = _GUARD_TOS_INT, + [_GUARD_NOS_OVERFLOWED_r22] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r00] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r11] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r33] = _GUARD_NOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r11] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r00] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r22] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r33] = _GUARD_TOS_OVERFLOWED, + [_BINARY_OP_MULTIPLY_INT_r21] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_ADD_INT_r21] = _BINARY_OP_ADD_INT, + [_BINARY_OP_SUBTRACT_INT_r21] = _BINARY_OP_SUBTRACT_INT, + [_GUARD_NOS_FLOAT_r22] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r00] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r11] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r33] = _GUARD_NOS_FLOAT, + [_GUARD_TOS_FLOAT_r11] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r00] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r22] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r33] = _GUARD_TOS_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r21] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_ADD_FLOAT_r21] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r21] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21] = _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21] = _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, + [_BINARY_OP_ADD_UNICODE_r21] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_INPLACE_ADD_UNICODE_r20] = _BINARY_OP_INPLACE_ADD_UNICODE, + [_GUARD_BINARY_OP_EXTEND_r22] = _GUARD_BINARY_OP_EXTEND, + [_BINARY_OP_EXTEND_r21] = _BINARY_OP_EXTEND, + [_BINARY_SLICE_r31] = _BINARY_SLICE, + [_STORE_SLICE_r30] = _STORE_SLICE, + [_BINARY_OP_SUBSCR_LIST_INT_r21] = _BINARY_OP_SUBSCR_LIST_INT, + [_BINARY_OP_SUBSCR_LIST_SLICE_r21] = _BINARY_OP_SUBSCR_LIST_SLICE, + [_BINARY_OP_SUBSCR_STR_INT_r21] = _BINARY_OP_SUBSCR_STR_INT, + [_GUARD_NOS_TUPLE_r22] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r00] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r11] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r33] = _GUARD_NOS_TUPLE, + [_GUARD_TOS_TUPLE_r11] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r00] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r22] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r33] = _GUARD_TOS_TUPLE, + [_BINARY_OP_SUBSCR_TUPLE_INT_r21] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_GUARD_NOS_DICT_r22] = _GUARD_NOS_DICT, + [_GUARD_NOS_DICT_r00] = _GUARD_NOS_DICT, + [_GUARD_NOS_DICT_r11] = _GUARD_NOS_DICT, + [_GUARD_NOS_DICT_r33] = _GUARD_NOS_DICT, + [_GUARD_TOS_DICT_r11] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r00] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r22] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r33] = _GUARD_TOS_DICT, + [_BINARY_OP_SUBSCR_DICT_r21] = _BINARY_OP_SUBSCR_DICT, + [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = _BINARY_OP_SUBSCR_CHECK_FUNC, + [_BINARY_OP_SUBSCR_INIT_CALL_r31] = _BINARY_OP_SUBSCR_INIT_CALL, + [_LIST_APPEND_r10] = _LIST_APPEND, + [_SET_ADD_r10] = _SET_ADD, + [_STORE_SUBSCR_r30] = _STORE_SUBSCR, + [_STORE_SUBSCR_LIST_INT_r30] = _STORE_SUBSCR_LIST_INT, + [_STORE_SUBSCR_DICT_r30] = _STORE_SUBSCR_DICT, + [_DELETE_SUBSCR_r20] = _DELETE_SUBSCR, + [_CALL_INTRINSIC_1_r11] = _CALL_INTRINSIC_1, + [_CALL_INTRINSIC_2_r21] = _CALL_INTRINSIC_2, + [_RETURN_VALUE_r11] = _RETURN_VALUE, + [_GET_AITER_r11] = _GET_AITER, + [_GET_ANEXT_r12] = _GET_ANEXT, + [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_SEND_GEN_FRAME_r22] = _SEND_GEN_FRAME, + [_YIELD_VALUE_r11] = _YIELD_VALUE, + [_POP_EXCEPT_r10] = _POP_EXCEPT, + [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, + [_LOAD_COMMON_CONSTANT_r12] = _LOAD_COMMON_CONSTANT, + [_LOAD_COMMON_CONSTANT_r23] = _LOAD_COMMON_CONSTANT, + [_LOAD_BUILD_CLASS_r01] = _LOAD_BUILD_CLASS, + [_STORE_NAME_r10] = _STORE_NAME, + [_DELETE_NAME_r00] = _DELETE_NAME, + [_UNPACK_SEQUENCE_r10] = _UNPACK_SEQUENCE, + [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_TWO_TUPLE, + [_UNPACK_SEQUENCE_TUPLE_r10] = _UNPACK_SEQUENCE_TUPLE, + [_UNPACK_SEQUENCE_LIST_r10] = _UNPACK_SEQUENCE_LIST, + [_UNPACK_EX_r10] = _UNPACK_EX, + [_STORE_ATTR_r20] = _STORE_ATTR, + [_DELETE_ATTR_r10] = _DELETE_ATTR, + [_STORE_GLOBAL_r10] = _STORE_GLOBAL, + [_DELETE_GLOBAL_r00] = _DELETE_GLOBAL, + [_LOAD_LOCALS_r01] = _LOAD_LOCALS, + [_LOAD_NAME_r01] = _LOAD_NAME, + [_LOAD_GLOBAL_r00] = _LOAD_GLOBAL, + [_PUSH_NULL_CONDITIONAL_r00] = _PUSH_NULL_CONDITIONAL, + [_GUARD_GLOBALS_VERSION_r00] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r11] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r22] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r33] = _GUARD_GLOBALS_VERSION, + [_LOAD_GLOBAL_MODULE_r01] = _LOAD_GLOBAL_MODULE, + [_LOAD_GLOBAL_BUILTINS_r01] = _LOAD_GLOBAL_BUILTINS, + [_DELETE_FAST_r00] = _DELETE_FAST, + [_MAKE_CELL_r00] = _MAKE_CELL, + [_DELETE_DEREF_r00] = _DELETE_DEREF, + [_LOAD_FROM_DICT_OR_DEREF_r11] = _LOAD_FROM_DICT_OR_DEREF, + [_LOAD_DEREF_r01] = _LOAD_DEREF, + [_STORE_DEREF_r10] = _STORE_DEREF, + [_COPY_FREE_VARS_r00] = _COPY_FREE_VARS, + [_BUILD_STRING_r01] = _BUILD_STRING, + [_BUILD_INTERPOLATION_r01] = _BUILD_INTERPOLATION, + [_BUILD_TEMPLATE_r21] = _BUILD_TEMPLATE, + [_BUILD_TUPLE_r01] = _BUILD_TUPLE, + [_BUILD_LIST_r01] = _BUILD_LIST, + [_LIST_EXTEND_r10] = _LIST_EXTEND, + [_SET_UPDATE_r10] = _SET_UPDATE, + [_BUILD_SET_r01] = _BUILD_SET, + [_BUILD_MAP_r01] = _BUILD_MAP, + [_SETUP_ANNOTATIONS_r00] = _SETUP_ANNOTATIONS, + [_DICT_UPDATE_r10] = _DICT_UPDATE, + [_DICT_MERGE_r10] = _DICT_MERGE, + [_MAP_ADD_r20] = _MAP_ADD, + [_LOAD_SUPER_ATTR_ATTR_r31] = _LOAD_SUPER_ATTR_ATTR, + [_LOAD_SUPER_ATTR_METHOD_r32] = _LOAD_SUPER_ATTR_METHOD, + [_LOAD_ATTR_r10] = _LOAD_ATTR, + [_GUARD_TYPE_VERSION_r11] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r00] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r22] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r33] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_AND_LOCK_r11] = _GUARD_TYPE_VERSION_AND_LOCK, + [_GUARD_TYPE_VERSION_AND_LOCK_r00] = _GUARD_TYPE_VERSION_AND_LOCK, + [_GUARD_TYPE_VERSION_AND_LOCK_r22] = _GUARD_TYPE_VERSION_AND_LOCK, + [_GUARD_TYPE_VERSION_AND_LOCK_r33] = _GUARD_TYPE_VERSION_AND_LOCK, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r00] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r33] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_LOAD_ATTR_INSTANCE_VALUE_r11] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_MODULE_r11] = _LOAD_ATTR_MODULE, + [_LOAD_ATTR_WITH_HINT_r11] = _LOAD_ATTR_WITH_HINT, + [_LOAD_ATTR_SLOT_r11] = _LOAD_ATTR_SLOT, + [_CHECK_ATTR_CLASS_r11] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r00] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r22] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r33] = _CHECK_ATTR_CLASS, + [_LOAD_ATTR_CLASS_r11] = _LOAD_ATTR_CLASS, + [_LOAD_ATTR_PROPERTY_FRAME_r11] = _LOAD_ATTR_PROPERTY_FRAME, + [_GUARD_DORV_NO_DICT_r11] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r00] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r22] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r33] = _GUARD_DORV_NO_DICT, + [_STORE_ATTR_INSTANCE_VALUE_r20] = _STORE_ATTR_INSTANCE_VALUE, + [_STORE_ATTR_WITH_HINT_r20] = _STORE_ATTR_WITH_HINT, + [_STORE_ATTR_SLOT_r20] = _STORE_ATTR_SLOT, + [_COMPARE_OP_r21] = _COMPARE_OP, + [_COMPARE_OP_FLOAT_r21] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_INT_r21] = _COMPARE_OP_INT, + [_COMPARE_OP_STR_r21] = _COMPARE_OP_STR, + [_IS_OP_r21] = _IS_OP, + [_CONTAINS_OP_r21] = _CONTAINS_OP, + [_GUARD_TOS_ANY_SET_r11] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r00] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r22] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r33] = _GUARD_TOS_ANY_SET, + [_CONTAINS_OP_SET_r21] = _CONTAINS_OP_SET, + [_CONTAINS_OP_DICT_r21] = _CONTAINS_OP_DICT, + [_CHECK_EG_MATCH_r22] = _CHECK_EG_MATCH, + [_CHECK_EXC_MATCH_r22] = _CHECK_EXC_MATCH, + [_IMPORT_NAME_r21] = _IMPORT_NAME, + [_IMPORT_FROM_r12] = _IMPORT_FROM, + [_IS_NONE_r11] = _IS_NONE, + [_GET_LEN_r12] = _GET_LEN, + [_MATCH_CLASS_r31] = _MATCH_CLASS, + [_MATCH_MAPPING_r12] = _MATCH_MAPPING, + [_MATCH_MAPPING_r01] = _MATCH_MAPPING, + [_MATCH_MAPPING_r23] = _MATCH_MAPPING, + [_MATCH_SEQUENCE_r12] = _MATCH_SEQUENCE, + [_MATCH_SEQUENCE_r01] = _MATCH_SEQUENCE, + [_MATCH_SEQUENCE_r23] = _MATCH_SEQUENCE, + [_MATCH_KEYS_r23] = _MATCH_KEYS, + [_GET_ITER_r12] = _GET_ITER, + [_GET_YIELD_FROM_ITER_r11] = _GET_YIELD_FROM_ITER, + [_FOR_ITER_TIER_TWO_r23] = _FOR_ITER_TIER_TWO, + [_ITER_CHECK_LIST_r22] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r00] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r11] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r33] = _ITER_CHECK_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r22] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r00] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r11] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r33] = _GUARD_NOT_EXHAUSTED_LIST, + [_ITER_NEXT_LIST_TIER_TWO_r23] = _ITER_NEXT_LIST_TIER_TWO, + [_ITER_CHECK_TUPLE_r22] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r00] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r11] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r33] = _ITER_CHECK_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r00] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r11] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_ITER_NEXT_TUPLE_r23] = _ITER_NEXT_TUPLE, + [_ITER_CHECK_RANGE_r22] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r00] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r11] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r33] = _ITER_CHECK_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r22] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r00] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r11] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r33] = _GUARD_NOT_EXHAUSTED_RANGE, + [_ITER_NEXT_RANGE_r23] = _ITER_NEXT_RANGE, + [_FOR_ITER_GEN_FRAME_r23] = _FOR_ITER_GEN_FRAME, + [_INSERT_NULL_r10] = _INSERT_NULL, + [_LOAD_SPECIAL_r00] = _LOAD_SPECIAL, + [_WITH_EXCEPT_START_r33] = _WITH_EXCEPT_START, + [_PUSH_EXC_INFO_r12] = _PUSH_EXC_INFO, + [_PUSH_EXC_INFO_r01] = _PUSH_EXC_INFO, + [_PUSH_EXC_INFO_r23] = _PUSH_EXC_INFO, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, + [_GUARD_KEYS_VERSION_r11] = _GUARD_KEYS_VERSION, + [_GUARD_KEYS_VERSION_r00] = _GUARD_KEYS_VERSION, + [_GUARD_KEYS_VERSION_r22] = _GUARD_KEYS_VERSION, + [_GUARD_KEYS_VERSION_r33] = _GUARD_KEYS_VERSION, + [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_WITH_VALUES_r01] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_NO_DICT_r12] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_METHOD_NO_DICT_r01] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_METHOD_NO_DICT_r23] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11] = _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r11] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r00] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r22] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r33] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r12] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r01] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r23] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_MAYBE_EXPAND_METHOD_r00] = _MAYBE_EXPAND_METHOD, + [_PY_FRAME_GENERAL_r01] = _PY_FRAME_GENERAL, + [_CHECK_FUNCTION_VERSION_r00] = _CHECK_FUNCTION_VERSION, + [_CHECK_FUNCTION_VERSION_INLINE_r00] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r11] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r22] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r33] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_METHOD_VERSION_r00] = _CHECK_METHOD_VERSION, + [_EXPAND_METHOD_r00] = _EXPAND_METHOD, + [_CHECK_IS_NOT_PY_CALLABLE_r00] = _CHECK_IS_NOT_PY_CALLABLE, + [_CALL_NON_PY_GENERAL_r01] = _CALL_NON_PY_GENERAL, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00] = _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00] = _INIT_CALL_BOUND_METHOD_EXACT_ARGS, + [_CHECK_PEP_523_r00] = _CHECK_PEP_523, + [_CHECK_PEP_523_r11] = _CHECK_PEP_523, + [_CHECK_PEP_523_r22] = _CHECK_PEP_523, + [_CHECK_PEP_523_r33] = _CHECK_PEP_523, + [_CHECK_FUNCTION_EXACT_ARGS_r00] = _CHECK_FUNCTION_EXACT_ARGS, + [_CHECK_STACK_SPACE_r00] = _CHECK_STACK_SPACE, + [_CHECK_RECURSION_REMAINING_r00] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r11] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r22] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r33] = _CHECK_RECURSION_REMAINING, + [_INIT_CALL_PY_EXACT_ARGS_0_r01] = _INIT_CALL_PY_EXACT_ARGS_0, + [_INIT_CALL_PY_EXACT_ARGS_1_r01] = _INIT_CALL_PY_EXACT_ARGS_1, + [_INIT_CALL_PY_EXACT_ARGS_2_r01] = _INIT_CALL_PY_EXACT_ARGS_2, + [_INIT_CALL_PY_EXACT_ARGS_3_r01] = _INIT_CALL_PY_EXACT_ARGS_3, + [_INIT_CALL_PY_EXACT_ARGS_4_r01] = _INIT_CALL_PY_EXACT_ARGS_4, + [_INIT_CALL_PY_EXACT_ARGS_r01] = _INIT_CALL_PY_EXACT_ARGS, + [_PUSH_FRAME_r10] = _PUSH_FRAME, + [_GUARD_NOS_NULL_r22] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r00] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r11] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r33] = _GUARD_NOS_NULL, + [_GUARD_NOS_NOT_NULL_r22] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r00] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r11] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r33] = _GUARD_NOS_NOT_NULL, + [_GUARD_THIRD_NULL_r33] = _GUARD_THIRD_NULL, + [_GUARD_CALLABLE_TYPE_1_r33] = _GUARD_CALLABLE_TYPE_1, + [_CALL_TYPE_1_r31] = _CALL_TYPE_1, + [_GUARD_CALLABLE_STR_1_r33] = _GUARD_CALLABLE_STR_1, + [_CALL_STR_1_r31] = _CALL_STR_1, + [_GUARD_CALLABLE_TUPLE_1_r33] = _GUARD_CALLABLE_TUPLE_1, + [_CALL_TUPLE_1_r31] = _CALL_TUPLE_1, + [_CHECK_AND_ALLOCATE_OBJECT_r00] = _CHECK_AND_ALLOCATE_OBJECT, + [_CREATE_INIT_FRAME_r01] = _CREATE_INIT_FRAME, + [_EXIT_INIT_CHECK_r10] = _EXIT_INIT_CHECK, + [_CALL_BUILTIN_CLASS_r01] = _CALL_BUILTIN_CLASS, + [_CALL_BUILTIN_O_r01] = _CALL_BUILTIN_O, + [_CALL_BUILTIN_FAST_r01] = _CALL_BUILTIN_FAST, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01] = _CALL_BUILTIN_FAST_WITH_KEYWORDS, + [_GUARD_CALLABLE_LEN_r33] = _GUARD_CALLABLE_LEN, + [_CALL_LEN_r31] = _CALL_LEN, + [_GUARD_CALLABLE_ISINSTANCE_r33] = _GUARD_CALLABLE_ISINSTANCE, + [_CALL_ISINSTANCE_r31] = _CALL_ISINSTANCE, + [_GUARD_CALLABLE_LIST_APPEND_r33] = _GUARD_CALLABLE_LIST_APPEND, + [_CALL_LIST_APPEND_r30] = _CALL_LIST_APPEND, + [_CALL_METHOD_DESCRIPTOR_O_r01] = _CALL_METHOD_DESCRIPTOR_O, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_NOARGS_r01] = _CALL_METHOD_DESCRIPTOR_NOARGS, + [_CALL_METHOD_DESCRIPTOR_FAST_r01] = _CALL_METHOD_DESCRIPTOR_FAST, + [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, + [_PY_FRAME_KW_r11] = _PY_FRAME_KW, + [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, + [_CHECK_METHOD_VERSION_KW_r11] = _CHECK_METHOD_VERSION_KW, + [_EXPAND_METHOD_KW_r11] = _EXPAND_METHOD_KW, + [_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = _CHECK_IS_NOT_PY_CALLABLE_KW, + [_CALL_KW_NON_PY_r11] = _CALL_KW_NON_PY, + [_MAKE_CALLARGS_A_TUPLE_r33] = _MAKE_CALLARGS_A_TUPLE, + [_MAKE_FUNCTION_r11] = _MAKE_FUNCTION, + [_SET_FUNCTION_ATTRIBUTE_r21] = _SET_FUNCTION_ATTRIBUTE, + [_RETURN_GENERATOR_r01] = _RETURN_GENERATOR, + [_BUILD_SLICE_r01] = _BUILD_SLICE, + [_CONVERT_VALUE_r11] = _CONVERT_VALUE, + [_FORMAT_SIMPLE_r11] = _FORMAT_SIMPLE, + [_FORMAT_WITH_SPEC_r21] = _FORMAT_WITH_SPEC, + [_COPY_1_r12] = _COPY_1, + [_COPY_1_r01] = _COPY_1, + [_COPY_1_r23] = _COPY_1, + [_COPY_2_r23] = _COPY_2, + [_COPY_3_r33] = _COPY_3, + [_COPY_r01] = _COPY, + [_BINARY_OP_r21] = _BINARY_OP, + [_SWAP_2_r22] = _SWAP_2, + [_SWAP_2_r00] = _SWAP_2, + [_SWAP_2_r11] = _SWAP_2, + [_SWAP_2_r33] = _SWAP_2, + [_SWAP_3_r33] = _SWAP_3, + [_SWAP_r11] = _SWAP, + [_GUARD_IS_TRUE_POP_r10] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r21] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r32] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_FALSE_POP_r10] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r21] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r32] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_NONE_POP_r10] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r21] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r32] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NOT_NONE_POP_r10] = _GUARD_IS_NOT_NONE_POP, + [_JUMP_TO_TOP_r00] = _JUMP_TO_TOP, + [_SET_IP_r00] = _SET_IP, + [_SET_IP_r11] = _SET_IP, + [_SET_IP_r22] = _SET_IP, + [_SET_IP_r33] = _SET_IP, + [_CHECK_STACK_SPACE_OPERAND_r00] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r11] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r22] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r33] = _CHECK_STACK_SPACE_OPERAND, + [_SAVE_RETURN_OFFSET_r00] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r11] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r22] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r33] = _SAVE_RETURN_OFFSET, + [_EXIT_TRACE_r00] = _EXIT_TRACE, + [_EXIT_TRACE_r10] = _EXIT_TRACE, + [_EXIT_TRACE_r20] = _EXIT_TRACE, + [_EXIT_TRACE_r30] = _EXIT_TRACE, + [_CHECK_VALIDITY_r00] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r11] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r22] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r33] = _CHECK_VALIDITY, + [_LOAD_CONST_INLINE_r01] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_r12] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_r23] = _LOAD_CONST_INLINE, + [_POP_TOP_LOAD_CONST_INLINE_r11] = _POP_TOP_LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_BORROW_r01] = _LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_INLINE_BORROW_r12] = _LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_INLINE_BORROW_r23] = _LOAD_CONST_INLINE_BORROW, + [_POP_CALL_r20] = _POP_CALL, + [_POP_CALL_ONE_r30] = _POP_CALL_ONE, + [_POP_CALL_TWO_r30] = _POP_CALL_TWO, + [_POP_TOP_LOAD_CONST_INLINE_BORROW_r11] = _POP_TOP_LOAD_CONST_INLINE_BORROW, + [_POP_TWO_LOAD_CONST_INLINE_BORROW_r21] = _POP_TWO_LOAD_CONST_INLINE_BORROW, + [_POP_CALL_LOAD_CONST_INLINE_BORROW_r21] = _POP_CALL_LOAD_CONST_INLINE_BORROW, + [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31] = _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, + [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31] = _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_UNDER_INLINE_r12] = _LOAD_CONST_UNDER_INLINE, + [_LOAD_CONST_UNDER_INLINE_r01] = _LOAD_CONST_UNDER_INLINE, + [_LOAD_CONST_UNDER_INLINE_r23] = _LOAD_CONST_UNDER_INLINE, + [_LOAD_CONST_UNDER_INLINE_BORROW_r12] = _LOAD_CONST_UNDER_INLINE_BORROW, + [_LOAD_CONST_UNDER_INLINE_BORROW_r01] = _LOAD_CONST_UNDER_INLINE_BORROW, + [_LOAD_CONST_UNDER_INLINE_BORROW_r23] = _LOAD_CONST_UNDER_INLINE_BORROW, + [_CHECK_FUNCTION_r00] = _CHECK_FUNCTION, + [_CHECK_FUNCTION_r11] = _CHECK_FUNCTION, + [_CHECK_FUNCTION_r22] = _CHECK_FUNCTION, + [_CHECK_FUNCTION_r33] = _CHECK_FUNCTION, + [_START_EXECUTOR_r00] = _START_EXECUTOR, + [_MAKE_WARM_r00] = _MAKE_WARM, + [_MAKE_WARM_r11] = _MAKE_WARM, + [_MAKE_WARM_r22] = _MAKE_WARM, + [_MAKE_WARM_r33] = _MAKE_WARM, + [_FATAL_ERROR_r00] = _FATAL_ERROR, + [_FATAL_ERROR_r11] = _FATAL_ERROR, + [_FATAL_ERROR_r22] = _FATAL_ERROR, + [_FATAL_ERROR_r33] = _FATAL_ERROR, + [_DEOPT_r00] = _DEOPT, + [_DEOPT_r10] = _DEOPT, + [_DEOPT_r20] = _DEOPT, + [_DEOPT_r30] = _DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r00] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r10] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r20] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r30] = _HANDLE_PENDING_AND_DEOPT, + [_ERROR_POP_N_r00] = _ERROR_POP_N, + [_TIER2_RESUME_CHECK_r00] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r11] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r22] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r33] = _TIER2_RESUME_CHECK, + [_SPILL_OR_RELOAD_r01] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r02] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r03] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r10] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r12] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r13] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r20] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r21] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r23] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r30] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r31] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r32] = _SPILL_OR_RELOAD, + [_COLD_EXIT_r00] = _COLD_EXIT, +}; + +const uint16_t _PyUop_SpillsAndReloads[4][4] = { + [0][1] = _SPILL_OR_RELOAD_r01, + [0][2] = _SPILL_OR_RELOAD_r02, + [0][3] = _SPILL_OR_RELOAD_r03, + [1][0] = _SPILL_OR_RELOAD_r10, + [1][2] = _SPILL_OR_RELOAD_r12, + [1][3] = _SPILL_OR_RELOAD_r13, + [2][0] = _SPILL_OR_RELOAD_r20, + [2][1] = _SPILL_OR_RELOAD_r21, + [2][3] = _SPILL_OR_RELOAD_r23, + [3][0] = _SPILL_OR_RELOAD_r30, + [3][1] = _SPILL_OR_RELOAD_r31, + [3][2] = _SPILL_OR_RELOAD_r32, +}; + +const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP] = "_BINARY_OP", + [_BINARY_OP_r21] = "_BINARY_OP_r21", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", + [_BINARY_OP_ADD_FLOAT_r21] = "_BINARY_OP_ADD_FLOAT_r21", [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", + [_BINARY_OP_ADD_INT_r21] = "_BINARY_OP_ADD_INT_r21", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_ADD_UNICODE_r21] = "_BINARY_OP_ADD_UNICODE_r21", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", + [_BINARY_OP_EXTEND_r21] = "_BINARY_OP_EXTEND_r21", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", + [_BINARY_OP_INPLACE_ADD_UNICODE_r20] = "_BINARY_OP_INPLACE_ADD_UNICODE_r20", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", + [_BINARY_OP_MULTIPLY_FLOAT_r21] = "_BINARY_OP_MULTIPLY_FLOAT_r21", [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", + [_BINARY_OP_MULTIPLY_INT_r21] = "_BINARY_OP_MULTIPLY_INT_r21", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", + [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = "_BINARY_OP_SUBSCR_CHECK_FUNC_r23", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", + [_BINARY_OP_SUBSCR_DICT_r21] = "_BINARY_OP_SUBSCR_DICT_r21", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", + [_BINARY_OP_SUBSCR_INIT_CALL_r31] = "_BINARY_OP_SUBSCR_INIT_CALL_r31", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", + [_BINARY_OP_SUBSCR_LIST_INT_r21] = "_BINARY_OP_SUBSCR_LIST_INT_r21", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", + [_BINARY_OP_SUBSCR_LIST_SLICE_r21] = "_BINARY_OP_SUBSCR_LIST_SLICE_r21", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", + [_BINARY_OP_SUBSCR_STR_INT_r21] = "_BINARY_OP_SUBSCR_STR_INT_r21", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", + [_BINARY_OP_SUBSCR_TUPLE_INT_r21] = "_BINARY_OP_SUBSCR_TUPLE_INT_r21", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", + [_BINARY_OP_SUBTRACT_FLOAT_r21] = "_BINARY_OP_SUBTRACT_FLOAT_r21", [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", + [_BINARY_OP_SUBTRACT_INT_r21] = "_BINARY_OP_SUBTRACT_INT_r21", [_BINARY_SLICE] = "_BINARY_SLICE", + [_BINARY_SLICE_r31] = "_BINARY_SLICE_r31", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", + [_BUILD_INTERPOLATION_r01] = "_BUILD_INTERPOLATION_r01", [_BUILD_LIST] = "_BUILD_LIST", + [_BUILD_LIST_r01] = "_BUILD_LIST_r01", [_BUILD_MAP] = "_BUILD_MAP", + [_BUILD_MAP_r01] = "_BUILD_MAP_r01", [_BUILD_SET] = "_BUILD_SET", + [_BUILD_SET_r01] = "_BUILD_SET_r01", [_BUILD_SLICE] = "_BUILD_SLICE", + [_BUILD_SLICE_r01] = "_BUILD_SLICE_r01", [_BUILD_STRING] = "_BUILD_STRING", + [_BUILD_STRING_r01] = "_BUILD_STRING_r01", [_BUILD_TEMPLATE] = "_BUILD_TEMPLATE", + [_BUILD_TEMPLATE_r21] = "_BUILD_TEMPLATE_r21", [_BUILD_TUPLE] = "_BUILD_TUPLE", + [_BUILD_TUPLE_r01] = "_BUILD_TUPLE_r01", [_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS", + [_CALL_BUILTIN_CLASS_r01] = "_CALL_BUILTIN_CLASS_r01", [_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST", + [_CALL_BUILTIN_FAST_r01] = "_CALL_BUILTIN_FAST_r01", [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS", + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01", [_CALL_BUILTIN_O] = "_CALL_BUILTIN_O", + [_CALL_BUILTIN_O_r01] = "_CALL_BUILTIN_O_r01", [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", + [_CALL_INTRINSIC_1_r11] = "_CALL_INTRINSIC_1_r11", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", + [_CALL_INTRINSIC_2_r21] = "_CALL_INTRINSIC_2_r21", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_ISINSTANCE_r31] = "_CALL_ISINSTANCE_r31", [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", + [_CALL_KW_NON_PY_r11] = "_CALL_KW_NON_PY_r11", [_CALL_LEN] = "_CALL_LEN", + [_CALL_LEN_r31] = "_CALL_LEN_r31", [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", + [_CALL_LIST_APPEND_r30] = "_CALL_LIST_APPEND_r30", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", + [_CALL_METHOD_DESCRIPTOR_FAST_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_r01", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", + [_CALL_METHOD_DESCRIPTOR_NOARGS_r01] = "_CALL_METHOD_DESCRIPTOR_NOARGS_r01", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_METHOD_DESCRIPTOR_O_r01] = "_CALL_METHOD_DESCRIPTOR_O_r01", [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", + [_CALL_NON_PY_GENERAL_r01] = "_CALL_NON_PY_GENERAL_r01", [_CALL_STR_1] = "_CALL_STR_1", + [_CALL_STR_1_r31] = "_CALL_STR_1_r31", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", + [_CALL_TUPLE_1_r31] = "_CALL_TUPLE_1_r31", [_CALL_TYPE_1] = "_CALL_TYPE_1", + [_CALL_TYPE_1_r31] = "_CALL_TYPE_1_r31", [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", + [_CHECK_AND_ALLOCATE_OBJECT_r00] = "_CHECK_AND_ALLOCATE_OBJECT_r00", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", + [_CHECK_ATTR_CLASS_r11] = "_CHECK_ATTR_CLASS_r11", + [_CHECK_ATTR_CLASS_r00] = "_CHECK_ATTR_CLASS_r00", + [_CHECK_ATTR_CLASS_r22] = "_CHECK_ATTR_CLASS_r22", + [_CHECK_ATTR_CLASS_r33] = "_CHECK_ATTR_CLASS_r33", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", + [_CHECK_ATTR_METHOD_LAZY_DICT_r11] = "_CHECK_ATTR_METHOD_LAZY_DICT_r11", + [_CHECK_ATTR_METHOD_LAZY_DICT_r00] = "_CHECK_ATTR_METHOD_LAZY_DICT_r00", + [_CHECK_ATTR_METHOD_LAZY_DICT_r22] = "_CHECK_ATTR_METHOD_LAZY_DICT_r22", + [_CHECK_ATTR_METHOD_LAZY_DICT_r33] = "_CHECK_ATTR_METHOD_LAZY_DICT_r33", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", + [_CHECK_EG_MATCH_r22] = "_CHECK_EG_MATCH_r22", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", + [_CHECK_EXC_MATCH_r22] = "_CHECK_EXC_MATCH_r22", [_CHECK_FUNCTION] = "_CHECK_FUNCTION", + [_CHECK_FUNCTION_r00] = "_CHECK_FUNCTION_r00", + [_CHECK_FUNCTION_r11] = "_CHECK_FUNCTION_r11", + [_CHECK_FUNCTION_r22] = "_CHECK_FUNCTION_r22", + [_CHECK_FUNCTION_r33] = "_CHECK_FUNCTION_r33", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_FUNCTION_EXACT_ARGS_r00] = "_CHECK_FUNCTION_EXACT_ARGS_r00", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_r00] = "_CHECK_FUNCTION_VERSION_r00", [_CHECK_FUNCTION_VERSION_INLINE] = "_CHECK_FUNCTION_VERSION_INLINE", + [_CHECK_FUNCTION_VERSION_INLINE_r00] = "_CHECK_FUNCTION_VERSION_INLINE_r00", + [_CHECK_FUNCTION_VERSION_INLINE_r11] = "_CHECK_FUNCTION_VERSION_INLINE_r11", + [_CHECK_FUNCTION_VERSION_INLINE_r22] = "_CHECK_FUNCTION_VERSION_INLINE_r22", + [_CHECK_FUNCTION_VERSION_INLINE_r33] = "_CHECK_FUNCTION_VERSION_INLINE_r33", [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", + [_CHECK_FUNCTION_VERSION_KW_r11] = "_CHECK_FUNCTION_VERSION_KW_r11", [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_IS_NOT_PY_CALLABLE_r00] = "_CHECK_IS_NOT_PY_CALLABLE_r00", [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", + [_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = "_CHECK_IS_NOT_PY_CALLABLE_KW_r11", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r11", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r00] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r00", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r22", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r33] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r33", [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_METHOD_VERSION_r00] = "_CHECK_METHOD_VERSION_r00", [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", + [_CHECK_METHOD_VERSION_KW_r11] = "_CHECK_METHOD_VERSION_KW_r11", [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_PEP_523_r00] = "_CHECK_PEP_523_r00", + [_CHECK_PEP_523_r11] = "_CHECK_PEP_523_r11", + [_CHECK_PEP_523_r22] = "_CHECK_PEP_523_r22", + [_CHECK_PEP_523_r33] = "_CHECK_PEP_523_r33", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", + [_CHECK_PERIODIC_r00] = "_CHECK_PERIODIC_r00", [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00", [_CHECK_RECURSION_REMAINING] = "_CHECK_RECURSION_REMAINING", + [_CHECK_RECURSION_REMAINING_r00] = "_CHECK_RECURSION_REMAINING_r00", + [_CHECK_RECURSION_REMAINING_r11] = "_CHECK_RECURSION_REMAINING_r11", + [_CHECK_RECURSION_REMAINING_r22] = "_CHECK_RECURSION_REMAINING_r22", + [_CHECK_RECURSION_REMAINING_r33] = "_CHECK_RECURSION_REMAINING_r33", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_STACK_SPACE_r00] = "_CHECK_STACK_SPACE_r00", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", + [_CHECK_STACK_SPACE_OPERAND_r00] = "_CHECK_STACK_SPACE_OPERAND_r00", + [_CHECK_STACK_SPACE_OPERAND_r11] = "_CHECK_STACK_SPACE_OPERAND_r11", + [_CHECK_STACK_SPACE_OPERAND_r22] = "_CHECK_STACK_SPACE_OPERAND_r22", + [_CHECK_STACK_SPACE_OPERAND_r33] = "_CHECK_STACK_SPACE_OPERAND_r33", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_CHECK_VALIDITY_r00] = "_CHECK_VALIDITY_r00", + [_CHECK_VALIDITY_r11] = "_CHECK_VALIDITY_r11", + [_CHECK_VALIDITY_r22] = "_CHECK_VALIDITY_r22", + [_CHECK_VALIDITY_r33] = "_CHECK_VALIDITY_r33", [_COLD_EXIT] = "_COLD_EXIT", + [_COLD_EXIT_r00] = "_COLD_EXIT_r00", [_COMPARE_OP] = "_COMPARE_OP", + [_COMPARE_OP_r21] = "_COMPARE_OP_r21", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", + [_COMPARE_OP_FLOAT_r21] = "_COMPARE_OP_FLOAT_r21", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", + [_COMPARE_OP_INT_r21] = "_COMPARE_OP_INT_r21", [_COMPARE_OP_STR] = "_COMPARE_OP_STR", + [_COMPARE_OP_STR_r21] = "_COMPARE_OP_STR_r21", [_CONTAINS_OP] = "_CONTAINS_OP", + [_CONTAINS_OP_r21] = "_CONTAINS_OP_r21", [_CONTAINS_OP_DICT] = "_CONTAINS_OP_DICT", + [_CONTAINS_OP_DICT_r21] = "_CONTAINS_OP_DICT_r21", [_CONTAINS_OP_SET] = "_CONTAINS_OP_SET", + [_CONTAINS_OP_SET_r21] = "_CONTAINS_OP_SET_r21", [_CONVERT_VALUE] = "_CONVERT_VALUE", + [_CONVERT_VALUE_r11] = "_CONVERT_VALUE_r11", [_COPY] = "_COPY", + [_COPY_r01] = "_COPY_r01", [_COPY_1] = "_COPY_1", + [_COPY_1_r12] = "_COPY_1_r12", + [_COPY_1_r01] = "_COPY_1_r01", + [_COPY_1_r23] = "_COPY_1_r23", [_COPY_2] = "_COPY_2", + [_COPY_2_r23] = "_COPY_2_r23", [_COPY_3] = "_COPY_3", + [_COPY_3_r33] = "_COPY_3_r33", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_COPY_FREE_VARS_r00] = "_COPY_FREE_VARS_r00", [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", + [_CREATE_INIT_FRAME_r01] = "_CREATE_INIT_FRAME_r01", [_DELETE_ATTR] = "_DELETE_ATTR", + [_DELETE_ATTR_r10] = "_DELETE_ATTR_r10", [_DELETE_DEREF] = "_DELETE_DEREF", + [_DELETE_DEREF_r00] = "_DELETE_DEREF_r00", [_DELETE_FAST] = "_DELETE_FAST", + [_DELETE_FAST_r00] = "_DELETE_FAST_r00", [_DELETE_GLOBAL] = "_DELETE_GLOBAL", + [_DELETE_GLOBAL_r00] = "_DELETE_GLOBAL_r00", [_DELETE_NAME] = "_DELETE_NAME", + [_DELETE_NAME_r00] = "_DELETE_NAME_r00", [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DELETE_SUBSCR_r20] = "_DELETE_SUBSCR_r20", [_DEOPT] = "_DEOPT", + [_DEOPT_r00] = "_DEOPT_r00", + [_DEOPT_r10] = "_DEOPT_r10", + [_DEOPT_r20] = "_DEOPT_r20", + [_DEOPT_r30] = "_DEOPT_r30", [_DICT_MERGE] = "_DICT_MERGE", + [_DICT_MERGE_r10] = "_DICT_MERGE_r10", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DICT_UPDATE_r10] = "_DICT_UPDATE_r10", [_END_FOR] = "_END_FOR", + [_END_FOR_r10] = "_END_FOR_r10", [_END_SEND] = "_END_SEND", + [_END_SEND_r21] = "_END_SEND_r21", [_ERROR_POP_N] = "_ERROR_POP_N", + [_ERROR_POP_N_r00] = "_ERROR_POP_N_r00", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", + [_EXIT_INIT_CHECK_r10] = "_EXIT_INIT_CHECK_r10", [_EXIT_TRACE] = "_EXIT_TRACE", + [_EXIT_TRACE_r00] = "_EXIT_TRACE_r00", + [_EXIT_TRACE_r10] = "_EXIT_TRACE_r10", + [_EXIT_TRACE_r20] = "_EXIT_TRACE_r20", + [_EXIT_TRACE_r30] = "_EXIT_TRACE_r30", [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_EXPAND_METHOD_r00] = "_EXPAND_METHOD_r00", [_EXPAND_METHOD_KW] = "_EXPAND_METHOD_KW", + [_EXPAND_METHOD_KW_r11] = "_EXPAND_METHOD_KW_r11", [_FATAL_ERROR] = "_FATAL_ERROR", + [_FATAL_ERROR_r00] = "_FATAL_ERROR_r00", + [_FATAL_ERROR_r11] = "_FATAL_ERROR_r11", + [_FATAL_ERROR_r22] = "_FATAL_ERROR_r22", + [_FATAL_ERROR_r33] = "_FATAL_ERROR_r33", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", + [_FORMAT_SIMPLE_r11] = "_FORMAT_SIMPLE_r11", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", + [_FORMAT_WITH_SPEC_r21] = "_FORMAT_WITH_SPEC_r21", [_FOR_ITER_GEN_FRAME] = "_FOR_ITER_GEN_FRAME", + [_FOR_ITER_GEN_FRAME_r23] = "_FOR_ITER_GEN_FRAME_r23", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", + [_FOR_ITER_TIER_TWO_r23] = "_FOR_ITER_TIER_TWO_r23", [_GET_AITER] = "_GET_AITER", + [_GET_AITER_r11] = "_GET_AITER_r11", [_GET_ANEXT] = "_GET_ANEXT", + [_GET_ANEXT_r12] = "_GET_ANEXT_r12", [_GET_AWAITABLE] = "_GET_AWAITABLE", + [_GET_AWAITABLE_r11] = "_GET_AWAITABLE_r11", [_GET_ITER] = "_GET_ITER", + [_GET_ITER_r12] = "_GET_ITER_r12", [_GET_LEN] = "_GET_LEN", + [_GET_LEN_r12] = "_GET_LEN_r12", [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", + [_GET_YIELD_FROM_ITER_r11] = "_GET_YIELD_FROM_ITER_r11", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", + [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", [_GUARD_CALLABLE_ISINSTANCE] = "_GUARD_CALLABLE_ISINSTANCE", + [_GUARD_CALLABLE_ISINSTANCE_r33] = "_GUARD_CALLABLE_ISINSTANCE_r33", [_GUARD_CALLABLE_LEN] = "_GUARD_CALLABLE_LEN", + [_GUARD_CALLABLE_LEN_r33] = "_GUARD_CALLABLE_LEN_r33", [_GUARD_CALLABLE_LIST_APPEND] = "_GUARD_CALLABLE_LIST_APPEND", + [_GUARD_CALLABLE_LIST_APPEND_r33] = "_GUARD_CALLABLE_LIST_APPEND_r33", [_GUARD_CALLABLE_STR_1] = "_GUARD_CALLABLE_STR_1", + [_GUARD_CALLABLE_STR_1_r33] = "_GUARD_CALLABLE_STR_1_r33", [_GUARD_CALLABLE_TUPLE_1] = "_GUARD_CALLABLE_TUPLE_1", + [_GUARD_CALLABLE_TUPLE_1_r33] = "_GUARD_CALLABLE_TUPLE_1_r33", [_GUARD_CALLABLE_TYPE_1] = "_GUARD_CALLABLE_TYPE_1", + [_GUARD_CALLABLE_TYPE_1_r33] = "_GUARD_CALLABLE_TYPE_1_r33", [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", + [_GUARD_DORV_NO_DICT_r11] = "_GUARD_DORV_NO_DICT_r11", + [_GUARD_DORV_NO_DICT_r00] = "_GUARD_DORV_NO_DICT_r00", + [_GUARD_DORV_NO_DICT_r22] = "_GUARD_DORV_NO_DICT_r22", + [_GUARD_DORV_NO_DICT_r33] = "_GUARD_DORV_NO_DICT_r33", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22", + [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_GLOBALS_VERSION_r00] = "_GUARD_GLOBALS_VERSION_r00", + [_GUARD_GLOBALS_VERSION_r11] = "_GUARD_GLOBALS_VERSION_r11", + [_GUARD_GLOBALS_VERSION_r22] = "_GUARD_GLOBALS_VERSION_r22", + [_GUARD_GLOBALS_VERSION_r33] = "_GUARD_GLOBALS_VERSION_r33", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", + [_GUARD_IS_FALSE_POP_r10] = "_GUARD_IS_FALSE_POP_r10", + [_GUARD_IS_FALSE_POP_r21] = "_GUARD_IS_FALSE_POP_r21", + [_GUARD_IS_FALSE_POP_r32] = "_GUARD_IS_FALSE_POP_r32", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", + [_GUARD_IS_NONE_POP_r10] = "_GUARD_IS_NONE_POP_r10", + [_GUARD_IS_NONE_POP_r21] = "_GUARD_IS_NONE_POP_r21", + [_GUARD_IS_NONE_POP_r32] = "_GUARD_IS_NONE_POP_r32", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", + [_GUARD_IS_NOT_NONE_POP_r10] = "_GUARD_IS_NOT_NONE_POP_r10", [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", + [_GUARD_IS_TRUE_POP_r10] = "_GUARD_IS_TRUE_POP_r10", + [_GUARD_IS_TRUE_POP_r21] = "_GUARD_IS_TRUE_POP_r21", + [_GUARD_IS_TRUE_POP_r32] = "_GUARD_IS_TRUE_POP_r32", [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", + [_GUARD_KEYS_VERSION_r11] = "_GUARD_KEYS_VERSION_r11", + [_GUARD_KEYS_VERSION_r00] = "_GUARD_KEYS_VERSION_r00", + [_GUARD_KEYS_VERSION_r22] = "_GUARD_KEYS_VERSION_r22", + [_GUARD_KEYS_VERSION_r33] = "_GUARD_KEYS_VERSION_r33", [_GUARD_NOS_DICT] = "_GUARD_NOS_DICT", + [_GUARD_NOS_DICT_r22] = "_GUARD_NOS_DICT_r22", + [_GUARD_NOS_DICT_r00] = "_GUARD_NOS_DICT_r00", + [_GUARD_NOS_DICT_r11] = "_GUARD_NOS_DICT_r11", + [_GUARD_NOS_DICT_r33] = "_GUARD_NOS_DICT_r33", [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", + [_GUARD_NOS_FLOAT_r22] = "_GUARD_NOS_FLOAT_r22", + [_GUARD_NOS_FLOAT_r00] = "_GUARD_NOS_FLOAT_r00", + [_GUARD_NOS_FLOAT_r11] = "_GUARD_NOS_FLOAT_r11", + [_GUARD_NOS_FLOAT_r33] = "_GUARD_NOS_FLOAT_r33", [_GUARD_NOS_INT] = "_GUARD_NOS_INT", + [_GUARD_NOS_INT_r22] = "_GUARD_NOS_INT_r22", + [_GUARD_NOS_INT_r00] = "_GUARD_NOS_INT_r00", + [_GUARD_NOS_INT_r11] = "_GUARD_NOS_INT_r11", + [_GUARD_NOS_INT_r33] = "_GUARD_NOS_INT_r33", [_GUARD_NOS_LIST] = "_GUARD_NOS_LIST", + [_GUARD_NOS_LIST_r22] = "_GUARD_NOS_LIST_r22", + [_GUARD_NOS_LIST_r00] = "_GUARD_NOS_LIST_r00", + [_GUARD_NOS_LIST_r11] = "_GUARD_NOS_LIST_r11", + [_GUARD_NOS_LIST_r33] = "_GUARD_NOS_LIST_r33", [_GUARD_NOS_NOT_NULL] = "_GUARD_NOS_NOT_NULL", + [_GUARD_NOS_NOT_NULL_r22] = "_GUARD_NOS_NOT_NULL_r22", + [_GUARD_NOS_NOT_NULL_r00] = "_GUARD_NOS_NOT_NULL_r00", + [_GUARD_NOS_NOT_NULL_r11] = "_GUARD_NOS_NOT_NULL_r11", + [_GUARD_NOS_NOT_NULL_r33] = "_GUARD_NOS_NOT_NULL_r33", [_GUARD_NOS_NULL] = "_GUARD_NOS_NULL", + [_GUARD_NOS_NULL_r22] = "_GUARD_NOS_NULL_r22", + [_GUARD_NOS_NULL_r00] = "_GUARD_NOS_NULL_r00", + [_GUARD_NOS_NULL_r11] = "_GUARD_NOS_NULL_r11", + [_GUARD_NOS_NULL_r33] = "_GUARD_NOS_NULL_r33", [_GUARD_NOS_OVERFLOWED] = "_GUARD_NOS_OVERFLOWED", + [_GUARD_NOS_OVERFLOWED_r22] = "_GUARD_NOS_OVERFLOWED_r22", + [_GUARD_NOS_OVERFLOWED_r00] = "_GUARD_NOS_OVERFLOWED_r00", + [_GUARD_NOS_OVERFLOWED_r11] = "_GUARD_NOS_OVERFLOWED_r11", + [_GUARD_NOS_OVERFLOWED_r33] = "_GUARD_NOS_OVERFLOWED_r33", [_GUARD_NOS_TUPLE] = "_GUARD_NOS_TUPLE", + [_GUARD_NOS_TUPLE_r22] = "_GUARD_NOS_TUPLE_r22", + [_GUARD_NOS_TUPLE_r00] = "_GUARD_NOS_TUPLE_r00", + [_GUARD_NOS_TUPLE_r11] = "_GUARD_NOS_TUPLE_r11", + [_GUARD_NOS_TUPLE_r33] = "_GUARD_NOS_TUPLE_r33", [_GUARD_NOS_UNICODE] = "_GUARD_NOS_UNICODE", + [_GUARD_NOS_UNICODE_r22] = "_GUARD_NOS_UNICODE_r22", + [_GUARD_NOS_UNICODE_r00] = "_GUARD_NOS_UNICODE_r00", + [_GUARD_NOS_UNICODE_r11] = "_GUARD_NOS_UNICODE_r11", + [_GUARD_NOS_UNICODE_r33] = "_GUARD_NOS_UNICODE_r33", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", + [_GUARD_NOT_EXHAUSTED_LIST_r22] = "_GUARD_NOT_EXHAUSTED_LIST_r22", + [_GUARD_NOT_EXHAUSTED_LIST_r00] = "_GUARD_NOT_EXHAUSTED_LIST_r00", + [_GUARD_NOT_EXHAUSTED_LIST_r11] = "_GUARD_NOT_EXHAUSTED_LIST_r11", + [_GUARD_NOT_EXHAUSTED_LIST_r33] = "_GUARD_NOT_EXHAUSTED_LIST_r33", [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", + [_GUARD_NOT_EXHAUSTED_RANGE_r22] = "_GUARD_NOT_EXHAUSTED_RANGE_r22", + [_GUARD_NOT_EXHAUSTED_RANGE_r00] = "_GUARD_NOT_EXHAUSTED_RANGE_r00", + [_GUARD_NOT_EXHAUSTED_RANGE_r11] = "_GUARD_NOT_EXHAUSTED_RANGE_r11", + [_GUARD_NOT_EXHAUSTED_RANGE_r33] = "_GUARD_NOT_EXHAUSTED_RANGE_r33", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = "_GUARD_NOT_EXHAUSTED_TUPLE_r22", + [_GUARD_NOT_EXHAUSTED_TUPLE_r00] = "_GUARD_NOT_EXHAUSTED_TUPLE_r00", + [_GUARD_NOT_EXHAUSTED_TUPLE_r11] = "_GUARD_NOT_EXHAUSTED_TUPLE_r11", + [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = "_GUARD_NOT_EXHAUSTED_TUPLE_r33", [_GUARD_THIRD_NULL] = "_GUARD_THIRD_NULL", + [_GUARD_THIRD_NULL_r33] = "_GUARD_THIRD_NULL_r33", [_GUARD_TOS_ANY_SET] = "_GUARD_TOS_ANY_SET", + [_GUARD_TOS_ANY_SET_r11] = "_GUARD_TOS_ANY_SET_r11", + [_GUARD_TOS_ANY_SET_r00] = "_GUARD_TOS_ANY_SET_r00", + [_GUARD_TOS_ANY_SET_r22] = "_GUARD_TOS_ANY_SET_r22", + [_GUARD_TOS_ANY_SET_r33] = "_GUARD_TOS_ANY_SET_r33", [_GUARD_TOS_DICT] = "_GUARD_TOS_DICT", + [_GUARD_TOS_DICT_r11] = "_GUARD_TOS_DICT_r11", + [_GUARD_TOS_DICT_r00] = "_GUARD_TOS_DICT_r00", + [_GUARD_TOS_DICT_r22] = "_GUARD_TOS_DICT_r22", + [_GUARD_TOS_DICT_r33] = "_GUARD_TOS_DICT_r33", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", + [_GUARD_TOS_FLOAT_r11] = "_GUARD_TOS_FLOAT_r11", + [_GUARD_TOS_FLOAT_r00] = "_GUARD_TOS_FLOAT_r00", + [_GUARD_TOS_FLOAT_r22] = "_GUARD_TOS_FLOAT_r22", + [_GUARD_TOS_FLOAT_r33] = "_GUARD_TOS_FLOAT_r33", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", + [_GUARD_TOS_INT_r11] = "_GUARD_TOS_INT_r11", + [_GUARD_TOS_INT_r00] = "_GUARD_TOS_INT_r00", + [_GUARD_TOS_INT_r22] = "_GUARD_TOS_INT_r22", + [_GUARD_TOS_INT_r33] = "_GUARD_TOS_INT_r33", [_GUARD_TOS_LIST] = "_GUARD_TOS_LIST", + [_GUARD_TOS_LIST_r11] = "_GUARD_TOS_LIST_r11", + [_GUARD_TOS_LIST_r00] = "_GUARD_TOS_LIST_r00", + [_GUARD_TOS_LIST_r22] = "_GUARD_TOS_LIST_r22", + [_GUARD_TOS_LIST_r33] = "_GUARD_TOS_LIST_r33", [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED", + [_GUARD_TOS_OVERFLOWED_r11] = "_GUARD_TOS_OVERFLOWED_r11", + [_GUARD_TOS_OVERFLOWED_r00] = "_GUARD_TOS_OVERFLOWED_r00", + [_GUARD_TOS_OVERFLOWED_r22] = "_GUARD_TOS_OVERFLOWED_r22", + [_GUARD_TOS_OVERFLOWED_r33] = "_GUARD_TOS_OVERFLOWED_r33", [_GUARD_TOS_SLICE] = "_GUARD_TOS_SLICE", + [_GUARD_TOS_SLICE_r11] = "_GUARD_TOS_SLICE_r11", + [_GUARD_TOS_SLICE_r00] = "_GUARD_TOS_SLICE_r00", + [_GUARD_TOS_SLICE_r22] = "_GUARD_TOS_SLICE_r22", + [_GUARD_TOS_SLICE_r33] = "_GUARD_TOS_SLICE_r33", [_GUARD_TOS_TUPLE] = "_GUARD_TOS_TUPLE", + [_GUARD_TOS_TUPLE_r11] = "_GUARD_TOS_TUPLE_r11", + [_GUARD_TOS_TUPLE_r00] = "_GUARD_TOS_TUPLE_r00", + [_GUARD_TOS_TUPLE_r22] = "_GUARD_TOS_TUPLE_r22", + [_GUARD_TOS_TUPLE_r33] = "_GUARD_TOS_TUPLE_r33", [_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE", + [_GUARD_TOS_UNICODE_r11] = "_GUARD_TOS_UNICODE_r11", + [_GUARD_TOS_UNICODE_r00] = "_GUARD_TOS_UNICODE_r00", + [_GUARD_TOS_UNICODE_r22] = "_GUARD_TOS_UNICODE_r22", + [_GUARD_TOS_UNICODE_r33] = "_GUARD_TOS_UNICODE_r33", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", + [_GUARD_TYPE_VERSION_r11] = "_GUARD_TYPE_VERSION_r11", + [_GUARD_TYPE_VERSION_r00] = "_GUARD_TYPE_VERSION_r00", + [_GUARD_TYPE_VERSION_r22] = "_GUARD_TYPE_VERSION_r22", + [_GUARD_TYPE_VERSION_r33] = "_GUARD_TYPE_VERSION_r33", [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", + [_GUARD_TYPE_VERSION_AND_LOCK_r11] = "_GUARD_TYPE_VERSION_AND_LOCK_r11", + [_GUARD_TYPE_VERSION_AND_LOCK_r00] = "_GUARD_TYPE_VERSION_AND_LOCK_r00", + [_GUARD_TYPE_VERSION_AND_LOCK_r22] = "_GUARD_TYPE_VERSION_AND_LOCK_r22", + [_GUARD_TYPE_VERSION_AND_LOCK_r33] = "_GUARD_TYPE_VERSION_AND_LOCK_r33", [_HANDLE_PENDING_AND_DEOPT] = "_HANDLE_PENDING_AND_DEOPT", + [_HANDLE_PENDING_AND_DEOPT_r00] = "_HANDLE_PENDING_AND_DEOPT_r00", + [_HANDLE_PENDING_AND_DEOPT_r10] = "_HANDLE_PENDING_AND_DEOPT_r10", + [_HANDLE_PENDING_AND_DEOPT_r20] = "_HANDLE_PENDING_AND_DEOPT_r20", + [_HANDLE_PENDING_AND_DEOPT_r30] = "_HANDLE_PENDING_AND_DEOPT_r30", [_IMPORT_FROM] = "_IMPORT_FROM", + [_IMPORT_FROM_r12] = "_IMPORT_FROM_r12", [_IMPORT_NAME] = "_IMPORT_NAME", + [_IMPORT_NAME_r21] = "_IMPORT_NAME_r21", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS_r01] = "_INIT_CALL_PY_EXACT_ARGS_r01", [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", + [_INIT_CALL_PY_EXACT_ARGS_0_r01] = "_INIT_CALL_PY_EXACT_ARGS_0_r01", [_INIT_CALL_PY_EXACT_ARGS_1] = "_INIT_CALL_PY_EXACT_ARGS_1", + [_INIT_CALL_PY_EXACT_ARGS_1_r01] = "_INIT_CALL_PY_EXACT_ARGS_1_r01", [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", + [_INIT_CALL_PY_EXACT_ARGS_2_r01] = "_INIT_CALL_PY_EXACT_ARGS_2_r01", [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", + [_INIT_CALL_PY_EXACT_ARGS_3_r01] = "_INIT_CALL_PY_EXACT_ARGS_3_r01", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", + [_INIT_CALL_PY_EXACT_ARGS_4_r01] = "_INIT_CALL_PY_EXACT_ARGS_4_r01", [_INSERT_NULL] = "_INSERT_NULL", + [_INSERT_NULL_r10] = "_INSERT_NULL_r10", [_IS_NONE] = "_IS_NONE", + [_IS_NONE_r11] = "_IS_NONE_r11", [_IS_OP] = "_IS_OP", + [_IS_OP_r21] = "_IS_OP_r21", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", + [_ITER_CHECK_LIST_r22] = "_ITER_CHECK_LIST_r22", + [_ITER_CHECK_LIST_r00] = "_ITER_CHECK_LIST_r00", + [_ITER_CHECK_LIST_r11] = "_ITER_CHECK_LIST_r11", + [_ITER_CHECK_LIST_r33] = "_ITER_CHECK_LIST_r33", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_CHECK_RANGE_r22] = "_ITER_CHECK_RANGE_r22", + [_ITER_CHECK_RANGE_r00] = "_ITER_CHECK_RANGE_r00", + [_ITER_CHECK_RANGE_r11] = "_ITER_CHECK_RANGE_r11", + [_ITER_CHECK_RANGE_r33] = "_ITER_CHECK_RANGE_r33", [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_CHECK_TUPLE_r22] = "_ITER_CHECK_TUPLE_r22", + [_ITER_CHECK_TUPLE_r00] = "_ITER_CHECK_TUPLE_r00", + [_ITER_CHECK_TUPLE_r11] = "_ITER_CHECK_TUPLE_r11", + [_ITER_CHECK_TUPLE_r33] = "_ITER_CHECK_TUPLE_r33", [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", + [_ITER_NEXT_LIST_TIER_TWO_r23] = "_ITER_NEXT_LIST_TIER_TWO_r23", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_ITER_NEXT_RANGE_r23] = "_ITER_NEXT_RANGE_r23", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_ITER_NEXT_TUPLE_r23] = "_ITER_NEXT_TUPLE_r23", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", + [_JUMP_TO_TOP_r00] = "_JUMP_TO_TOP_r00", [_LIST_APPEND] = "_LIST_APPEND", + [_LIST_APPEND_r10] = "_LIST_APPEND_r10", [_LIST_EXTEND] = "_LIST_EXTEND", + [_LIST_EXTEND_r10] = "_LIST_EXTEND_r10", [_LOAD_ATTR] = "_LOAD_ATTR", + [_LOAD_ATTR_r10] = "_LOAD_ATTR_r10", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_CLASS_r11] = "_LOAD_ATTR_CLASS_r11", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_INSTANCE_VALUE_r11] = "_LOAD_ATTR_INSTANCE_VALUE_r11", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", + [_LOAD_ATTR_METHOD_LAZY_DICT_r12] = "_LOAD_ATTR_METHOD_LAZY_DICT_r12", + [_LOAD_ATTR_METHOD_LAZY_DICT_r01] = "_LOAD_ATTR_METHOD_LAZY_DICT_r01", + [_LOAD_ATTR_METHOD_LAZY_DICT_r23] = "_LOAD_ATTR_METHOD_LAZY_DICT_r23", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", + [_LOAD_ATTR_METHOD_NO_DICT_r12] = "_LOAD_ATTR_METHOD_NO_DICT_r12", + [_LOAD_ATTR_METHOD_NO_DICT_r01] = "_LOAD_ATTR_METHOD_NO_DICT_r01", + [_LOAD_ATTR_METHOD_NO_DICT_r23] = "_LOAD_ATTR_METHOD_NO_DICT_r23", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = "_LOAD_ATTR_METHOD_WITH_VALUES_r12", + [_LOAD_ATTR_METHOD_WITH_VALUES_r01] = "_LOAD_ATTR_METHOD_WITH_VALUES_r01", + [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = "_LOAD_ATTR_METHOD_WITH_VALUES_r23", [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_LOAD_ATTR_MODULE_r11] = "_LOAD_ATTR_MODULE_r11", [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11", [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", + [_LOAD_ATTR_PROPERTY_FRAME_r11] = "_LOAD_ATTR_PROPERTY_FRAME_r11", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_SLOT_r11] = "_LOAD_ATTR_SLOT_r11", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", + [_LOAD_ATTR_WITH_HINT_r11] = "_LOAD_ATTR_WITH_HINT_r11", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", + [_LOAD_BUILD_CLASS_r01] = "_LOAD_BUILD_CLASS_r01", [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", + [_LOAD_COMMON_CONSTANT_r01] = "_LOAD_COMMON_CONSTANT_r01", + [_LOAD_COMMON_CONSTANT_r12] = "_LOAD_COMMON_CONSTANT_r12", + [_LOAD_COMMON_CONSTANT_r23] = "_LOAD_COMMON_CONSTANT_r23", [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_CONST_r01] = "_LOAD_CONST_r01", + [_LOAD_CONST_r12] = "_LOAD_CONST_r12", + [_LOAD_CONST_r23] = "_LOAD_CONST_r23", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", + [_LOAD_CONST_INLINE_r01] = "_LOAD_CONST_INLINE_r01", + [_LOAD_CONST_INLINE_r12] = "_LOAD_CONST_INLINE_r12", + [_LOAD_CONST_INLINE_r23] = "_LOAD_CONST_INLINE_r23", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", + [_LOAD_CONST_INLINE_BORROW_r01] = "_LOAD_CONST_INLINE_BORROW_r01", + [_LOAD_CONST_INLINE_BORROW_r12] = "_LOAD_CONST_INLINE_BORROW_r12", + [_LOAD_CONST_INLINE_BORROW_r23] = "_LOAD_CONST_INLINE_BORROW_r23", [_LOAD_CONST_UNDER_INLINE] = "_LOAD_CONST_UNDER_INLINE", + [_LOAD_CONST_UNDER_INLINE_r12] = "_LOAD_CONST_UNDER_INLINE_r12", + [_LOAD_CONST_UNDER_INLINE_r01] = "_LOAD_CONST_UNDER_INLINE_r01", + [_LOAD_CONST_UNDER_INLINE_r23] = "_LOAD_CONST_UNDER_INLINE_r23", [_LOAD_CONST_UNDER_INLINE_BORROW] = "_LOAD_CONST_UNDER_INLINE_BORROW", + [_LOAD_CONST_UNDER_INLINE_BORROW_r12] = "_LOAD_CONST_UNDER_INLINE_BORROW_r12", + [_LOAD_CONST_UNDER_INLINE_BORROW_r01] = "_LOAD_CONST_UNDER_INLINE_BORROW_r01", + [_LOAD_CONST_UNDER_INLINE_BORROW_r23] = "_LOAD_CONST_UNDER_INLINE_BORROW_r23", [_LOAD_DEREF] = "_LOAD_DEREF", + [_LOAD_DEREF_r01] = "_LOAD_DEREF_r01", [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_r01] = "_LOAD_FAST_r01", + [_LOAD_FAST_r12] = "_LOAD_FAST_r12", + [_LOAD_FAST_r23] = "_LOAD_FAST_r23", [_LOAD_FAST_0] = "_LOAD_FAST_0", + [_LOAD_FAST_0_r01] = "_LOAD_FAST_0_r01", + [_LOAD_FAST_0_r12] = "_LOAD_FAST_0_r12", + [_LOAD_FAST_0_r23] = "_LOAD_FAST_0_r23", [_LOAD_FAST_1] = "_LOAD_FAST_1", + [_LOAD_FAST_1_r01] = "_LOAD_FAST_1_r01", + [_LOAD_FAST_1_r12] = "_LOAD_FAST_1_r12", + [_LOAD_FAST_1_r23] = "_LOAD_FAST_1_r23", [_LOAD_FAST_2] = "_LOAD_FAST_2", + [_LOAD_FAST_2_r01] = "_LOAD_FAST_2_r01", + [_LOAD_FAST_2_r12] = "_LOAD_FAST_2_r12", + [_LOAD_FAST_2_r23] = "_LOAD_FAST_2_r23", [_LOAD_FAST_3] = "_LOAD_FAST_3", + [_LOAD_FAST_3_r01] = "_LOAD_FAST_3_r01", + [_LOAD_FAST_3_r12] = "_LOAD_FAST_3_r12", + [_LOAD_FAST_3_r23] = "_LOAD_FAST_3_r23", [_LOAD_FAST_4] = "_LOAD_FAST_4", + [_LOAD_FAST_4_r01] = "_LOAD_FAST_4_r01", + [_LOAD_FAST_4_r12] = "_LOAD_FAST_4_r12", + [_LOAD_FAST_4_r23] = "_LOAD_FAST_4_r23", [_LOAD_FAST_5] = "_LOAD_FAST_5", + [_LOAD_FAST_5_r01] = "_LOAD_FAST_5_r01", + [_LOAD_FAST_5_r12] = "_LOAD_FAST_5_r12", + [_LOAD_FAST_5_r23] = "_LOAD_FAST_5_r23", [_LOAD_FAST_6] = "_LOAD_FAST_6", + [_LOAD_FAST_6_r01] = "_LOAD_FAST_6_r01", + [_LOAD_FAST_6_r12] = "_LOAD_FAST_6_r12", + [_LOAD_FAST_6_r23] = "_LOAD_FAST_6_r23", [_LOAD_FAST_7] = "_LOAD_FAST_7", + [_LOAD_FAST_7_r01] = "_LOAD_FAST_7_r01", + [_LOAD_FAST_7_r12] = "_LOAD_FAST_7_r12", + [_LOAD_FAST_7_r23] = "_LOAD_FAST_7_r23", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", + [_LOAD_FAST_AND_CLEAR_r01] = "_LOAD_FAST_AND_CLEAR_r01", + [_LOAD_FAST_AND_CLEAR_r12] = "_LOAD_FAST_AND_CLEAR_r12", + [_LOAD_FAST_AND_CLEAR_r23] = "_LOAD_FAST_AND_CLEAR_r23", [_LOAD_FAST_BORROW] = "_LOAD_FAST_BORROW", + [_LOAD_FAST_BORROW_r01] = "_LOAD_FAST_BORROW_r01", + [_LOAD_FAST_BORROW_r12] = "_LOAD_FAST_BORROW_r12", + [_LOAD_FAST_BORROW_r23] = "_LOAD_FAST_BORROW_r23", [_LOAD_FAST_BORROW_0] = "_LOAD_FAST_BORROW_0", + [_LOAD_FAST_BORROW_0_r01] = "_LOAD_FAST_BORROW_0_r01", + [_LOAD_FAST_BORROW_0_r12] = "_LOAD_FAST_BORROW_0_r12", + [_LOAD_FAST_BORROW_0_r23] = "_LOAD_FAST_BORROW_0_r23", [_LOAD_FAST_BORROW_1] = "_LOAD_FAST_BORROW_1", + [_LOAD_FAST_BORROW_1_r01] = "_LOAD_FAST_BORROW_1_r01", + [_LOAD_FAST_BORROW_1_r12] = "_LOAD_FAST_BORROW_1_r12", + [_LOAD_FAST_BORROW_1_r23] = "_LOAD_FAST_BORROW_1_r23", [_LOAD_FAST_BORROW_2] = "_LOAD_FAST_BORROW_2", + [_LOAD_FAST_BORROW_2_r01] = "_LOAD_FAST_BORROW_2_r01", + [_LOAD_FAST_BORROW_2_r12] = "_LOAD_FAST_BORROW_2_r12", + [_LOAD_FAST_BORROW_2_r23] = "_LOAD_FAST_BORROW_2_r23", [_LOAD_FAST_BORROW_3] = "_LOAD_FAST_BORROW_3", + [_LOAD_FAST_BORROW_3_r01] = "_LOAD_FAST_BORROW_3_r01", + [_LOAD_FAST_BORROW_3_r12] = "_LOAD_FAST_BORROW_3_r12", + [_LOAD_FAST_BORROW_3_r23] = "_LOAD_FAST_BORROW_3_r23", [_LOAD_FAST_BORROW_4] = "_LOAD_FAST_BORROW_4", + [_LOAD_FAST_BORROW_4_r01] = "_LOAD_FAST_BORROW_4_r01", + [_LOAD_FAST_BORROW_4_r12] = "_LOAD_FAST_BORROW_4_r12", + [_LOAD_FAST_BORROW_4_r23] = "_LOAD_FAST_BORROW_4_r23", [_LOAD_FAST_BORROW_5] = "_LOAD_FAST_BORROW_5", + [_LOAD_FAST_BORROW_5_r01] = "_LOAD_FAST_BORROW_5_r01", + [_LOAD_FAST_BORROW_5_r12] = "_LOAD_FAST_BORROW_5_r12", + [_LOAD_FAST_BORROW_5_r23] = "_LOAD_FAST_BORROW_5_r23", [_LOAD_FAST_BORROW_6] = "_LOAD_FAST_BORROW_6", + [_LOAD_FAST_BORROW_6_r01] = "_LOAD_FAST_BORROW_6_r01", + [_LOAD_FAST_BORROW_6_r12] = "_LOAD_FAST_BORROW_6_r12", + [_LOAD_FAST_BORROW_6_r23] = "_LOAD_FAST_BORROW_6_r23", [_LOAD_FAST_BORROW_7] = "_LOAD_FAST_BORROW_7", - [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = "_LOAD_FAST_BORROW_LOAD_FAST_BORROW", + [_LOAD_FAST_BORROW_7_r01] = "_LOAD_FAST_BORROW_7_r01", + [_LOAD_FAST_BORROW_7_r12] = "_LOAD_FAST_BORROW_7_r12", + [_LOAD_FAST_BORROW_7_r23] = "_LOAD_FAST_BORROW_7_r23", [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", - [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", + [_LOAD_FAST_CHECK_r01] = "_LOAD_FAST_CHECK_r01", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", + [_LOAD_FROM_DICT_OR_DEREF_r11] = "_LOAD_FROM_DICT_OR_DEREF_r11", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", + [_LOAD_GLOBAL_r00] = "_LOAD_GLOBAL_r00", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_BUILTINS_r01] = "_LOAD_GLOBAL_BUILTINS_r01", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_GLOBAL_MODULE_r01] = "_LOAD_GLOBAL_MODULE_r01", [_LOAD_LOCALS] = "_LOAD_LOCALS", + [_LOAD_LOCALS_r01] = "_LOAD_LOCALS_r01", [_LOAD_NAME] = "_LOAD_NAME", + [_LOAD_NAME_r01] = "_LOAD_NAME_r01", [_LOAD_SMALL_INT] = "_LOAD_SMALL_INT", + [_LOAD_SMALL_INT_r01] = "_LOAD_SMALL_INT_r01", + [_LOAD_SMALL_INT_r12] = "_LOAD_SMALL_INT_r12", + [_LOAD_SMALL_INT_r23] = "_LOAD_SMALL_INT_r23", [_LOAD_SMALL_INT_0] = "_LOAD_SMALL_INT_0", + [_LOAD_SMALL_INT_0_r01] = "_LOAD_SMALL_INT_0_r01", + [_LOAD_SMALL_INT_0_r12] = "_LOAD_SMALL_INT_0_r12", + [_LOAD_SMALL_INT_0_r23] = "_LOAD_SMALL_INT_0_r23", [_LOAD_SMALL_INT_1] = "_LOAD_SMALL_INT_1", + [_LOAD_SMALL_INT_1_r01] = "_LOAD_SMALL_INT_1_r01", + [_LOAD_SMALL_INT_1_r12] = "_LOAD_SMALL_INT_1_r12", + [_LOAD_SMALL_INT_1_r23] = "_LOAD_SMALL_INT_1_r23", [_LOAD_SMALL_INT_2] = "_LOAD_SMALL_INT_2", + [_LOAD_SMALL_INT_2_r01] = "_LOAD_SMALL_INT_2_r01", + [_LOAD_SMALL_INT_2_r12] = "_LOAD_SMALL_INT_2_r12", + [_LOAD_SMALL_INT_2_r23] = "_LOAD_SMALL_INT_2_r23", [_LOAD_SMALL_INT_3] = "_LOAD_SMALL_INT_3", + [_LOAD_SMALL_INT_3_r01] = "_LOAD_SMALL_INT_3_r01", + [_LOAD_SMALL_INT_3_r12] = "_LOAD_SMALL_INT_3_r12", + [_LOAD_SMALL_INT_3_r23] = "_LOAD_SMALL_INT_3_r23", [_LOAD_SPECIAL] = "_LOAD_SPECIAL", + [_LOAD_SPECIAL_r00] = "_LOAD_SPECIAL_r00", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", + [_LOAD_SUPER_ATTR_ATTR_r31] = "_LOAD_SUPER_ATTR_ATTR_r31", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", + [_LOAD_SUPER_ATTR_METHOD_r32] = "_LOAD_SUPER_ATTR_METHOD_r32", [_MAKE_CALLARGS_A_TUPLE] = "_MAKE_CALLARGS_A_TUPLE", + [_MAKE_CALLARGS_A_TUPLE_r33] = "_MAKE_CALLARGS_A_TUPLE_r33", [_MAKE_CELL] = "_MAKE_CELL", + [_MAKE_CELL_r00] = "_MAKE_CELL_r00", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_FUNCTION_r11] = "_MAKE_FUNCTION_r11", [_MAKE_WARM] = "_MAKE_WARM", + [_MAKE_WARM_r00] = "_MAKE_WARM_r00", + [_MAKE_WARM_r11] = "_MAKE_WARM_r11", + [_MAKE_WARM_r22] = "_MAKE_WARM_r22", + [_MAKE_WARM_r33] = "_MAKE_WARM_r33", [_MAP_ADD] = "_MAP_ADD", + [_MAP_ADD_r20] = "_MAP_ADD_r20", [_MATCH_CLASS] = "_MATCH_CLASS", + [_MATCH_CLASS_r31] = "_MATCH_CLASS_r31", [_MATCH_KEYS] = "_MATCH_KEYS", + [_MATCH_KEYS_r23] = "_MATCH_KEYS_r23", [_MATCH_MAPPING] = "_MATCH_MAPPING", + [_MATCH_MAPPING_r12] = "_MATCH_MAPPING_r12", + [_MATCH_MAPPING_r01] = "_MATCH_MAPPING_r01", + [_MATCH_MAPPING_r23] = "_MATCH_MAPPING_r23", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_MATCH_SEQUENCE_r12] = "_MATCH_SEQUENCE_r12", + [_MATCH_SEQUENCE_r01] = "_MATCH_SEQUENCE_r01", + [_MATCH_SEQUENCE_r23] = "_MATCH_SEQUENCE_r23", [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", + [_MAYBE_EXPAND_METHOD_r00] = "_MAYBE_EXPAND_METHOD_r00", [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", + [_MAYBE_EXPAND_METHOD_KW_r11] = "_MAYBE_EXPAND_METHOD_KW_r11", [_NOP] = "_NOP", + [_NOP_r00] = "_NOP_r00", + [_NOP_r11] = "_NOP_r11", + [_NOP_r22] = "_NOP_r22", + [_NOP_r33] = "_NOP_r33", [_POP_CALL] = "_POP_CALL", + [_POP_CALL_r20] = "_POP_CALL_r20", [_POP_CALL_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_LOAD_CONST_INLINE_BORROW", + [_POP_CALL_LOAD_CONST_INLINE_BORROW_r21] = "_POP_CALL_LOAD_CONST_INLINE_BORROW_r21", [_POP_CALL_ONE] = "_POP_CALL_ONE", + [_POP_CALL_ONE_r30] = "_POP_CALL_ONE_r30", [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", + [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31] = "_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31", [_POP_CALL_TWO] = "_POP_CALL_TWO", + [_POP_CALL_TWO_r30] = "_POP_CALL_TWO_r30", [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", + [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31", [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_EXCEPT_r10] = "_POP_EXCEPT_r10", [_POP_ITER] = "_POP_ITER", + [_POP_ITER_r20] = "_POP_ITER_r20", [_POP_TOP] = "_POP_TOP", + [_POP_TOP_r10] = "_POP_TOP_r10", [_POP_TOP_FLOAT] = "_POP_TOP_FLOAT", + [_POP_TOP_FLOAT_r10] = "_POP_TOP_FLOAT_r10", + [_POP_TOP_FLOAT_r21] = "_POP_TOP_FLOAT_r21", + [_POP_TOP_FLOAT_r32] = "_POP_TOP_FLOAT_r32", [_POP_TOP_INT] = "_POP_TOP_INT", + [_POP_TOP_INT_r10] = "_POP_TOP_INT_r10", + [_POP_TOP_INT_r21] = "_POP_TOP_INT_r21", + [_POP_TOP_INT_r32] = "_POP_TOP_INT_r32", [_POP_TOP_LOAD_CONST_INLINE] = "_POP_TOP_LOAD_CONST_INLINE", + [_POP_TOP_LOAD_CONST_INLINE_r11] = "_POP_TOP_LOAD_CONST_INLINE_r11", [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", + [_POP_TOP_LOAD_CONST_INLINE_BORROW_r11] = "_POP_TOP_LOAD_CONST_INLINE_BORROW_r11", [_POP_TOP_NOP] = "_POP_TOP_NOP", + [_POP_TOP_NOP_r10] = "_POP_TOP_NOP_r10", + [_POP_TOP_NOP_r21] = "_POP_TOP_NOP_r21", + [_POP_TOP_NOP_r32] = "_POP_TOP_NOP_r32", [_POP_TOP_UNICODE] = "_POP_TOP_UNICODE", + [_POP_TOP_UNICODE_r10] = "_POP_TOP_UNICODE_r10", + [_POP_TOP_UNICODE_r21] = "_POP_TOP_UNICODE_r21", + [_POP_TOP_UNICODE_r32] = "_POP_TOP_UNICODE_r32", [_POP_TWO] = "_POP_TWO", + [_POP_TWO_r20] = "_POP_TWO_r20", [_POP_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_TWO_LOAD_CONST_INLINE_BORROW", + [_POP_TWO_LOAD_CONST_INLINE_BORROW_r21] = "_POP_TWO_LOAD_CONST_INLINE_BORROW_r21", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", + [_PUSH_EXC_INFO_r12] = "_PUSH_EXC_INFO_r12", + [_PUSH_EXC_INFO_r01] = "_PUSH_EXC_INFO_r01", + [_PUSH_EXC_INFO_r23] = "_PUSH_EXC_INFO_r23", [_PUSH_FRAME] = "_PUSH_FRAME", + [_PUSH_FRAME_r10] = "_PUSH_FRAME_r10", [_PUSH_NULL] = "_PUSH_NULL", + [_PUSH_NULL_r01] = "_PUSH_NULL_r01", + [_PUSH_NULL_r12] = "_PUSH_NULL_r12", + [_PUSH_NULL_r23] = "_PUSH_NULL_r23", [_PUSH_NULL_CONDITIONAL] = "_PUSH_NULL_CONDITIONAL", + [_PUSH_NULL_CONDITIONAL_r00] = "_PUSH_NULL_CONDITIONAL_r00", [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_PY_FRAME_GENERAL_r01] = "_PY_FRAME_GENERAL_r01", [_PY_FRAME_KW] = "_PY_FRAME_KW", + [_PY_FRAME_KW_r11] = "_PY_FRAME_KW_r11", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", + [_REPLACE_WITH_TRUE_r11] = "_REPLACE_WITH_TRUE_r11", [_RESUME_CHECK] = "_RESUME_CHECK", + [_RESUME_CHECK_r00] = "_RESUME_CHECK_r00", + [_RESUME_CHECK_r11] = "_RESUME_CHECK_r11", + [_RESUME_CHECK_r22] = "_RESUME_CHECK_r22", + [_RESUME_CHECK_r33] = "_RESUME_CHECK_r33", [_RETURN_GENERATOR] = "_RETURN_GENERATOR", + [_RETURN_GENERATOR_r01] = "_RETURN_GENERATOR_r01", [_RETURN_VALUE] = "_RETURN_VALUE", + [_RETURN_VALUE_r11] = "_RETURN_VALUE_r11", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SAVE_RETURN_OFFSET_r00] = "_SAVE_RETURN_OFFSET_r00", + [_SAVE_RETURN_OFFSET_r11] = "_SAVE_RETURN_OFFSET_r11", + [_SAVE_RETURN_OFFSET_r22] = "_SAVE_RETURN_OFFSET_r22", + [_SAVE_RETURN_OFFSET_r33] = "_SAVE_RETURN_OFFSET_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", + [_SEND_GEN_FRAME_r22] = "_SEND_GEN_FRAME_r22", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", + [_SETUP_ANNOTATIONS_r00] = "_SETUP_ANNOTATIONS_r00", [_SET_ADD] = "_SET_ADD", + [_SET_ADD_r10] = "_SET_ADD_r10", [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", + [_SET_FUNCTION_ATTRIBUTE_r21] = "_SET_FUNCTION_ATTRIBUTE_r21", [_SET_IP] = "_SET_IP", + [_SET_IP_r00] = "_SET_IP_r00", + [_SET_IP_r11] = "_SET_IP_r11", + [_SET_IP_r22] = "_SET_IP_r22", + [_SET_IP_r33] = "_SET_IP_r33", [_SET_UPDATE] = "_SET_UPDATE", + [_SET_UPDATE_r10] = "_SET_UPDATE_r10", + [_SPILL_OR_RELOAD] = "_SPILL_OR_RELOAD", + [_SPILL_OR_RELOAD_r01] = "_SPILL_OR_RELOAD_r01", + [_SPILL_OR_RELOAD_r02] = "_SPILL_OR_RELOAD_r02", + [_SPILL_OR_RELOAD_r03] = "_SPILL_OR_RELOAD_r03", + [_SPILL_OR_RELOAD_r10] = "_SPILL_OR_RELOAD_r10", + [_SPILL_OR_RELOAD_r12] = "_SPILL_OR_RELOAD_r12", + [_SPILL_OR_RELOAD_r13] = "_SPILL_OR_RELOAD_r13", + [_SPILL_OR_RELOAD_r20] = "_SPILL_OR_RELOAD_r20", + [_SPILL_OR_RELOAD_r21] = "_SPILL_OR_RELOAD_r21", + [_SPILL_OR_RELOAD_r23] = "_SPILL_OR_RELOAD_r23", + [_SPILL_OR_RELOAD_r30] = "_SPILL_OR_RELOAD_r30", + [_SPILL_OR_RELOAD_r31] = "_SPILL_OR_RELOAD_r31", + [_SPILL_OR_RELOAD_r32] = "_SPILL_OR_RELOAD_r32", [_START_EXECUTOR] = "_START_EXECUTOR", + [_START_EXECUTOR_r00] = "_START_EXECUTOR_r00", [_STORE_ATTR] = "_STORE_ATTR", + [_STORE_ATTR_r20] = "_STORE_ATTR_r20", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", + [_STORE_ATTR_INSTANCE_VALUE_r20] = "_STORE_ATTR_INSTANCE_VALUE_r20", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_ATTR_SLOT_r20] = "_STORE_ATTR_SLOT_r20", [_STORE_ATTR_WITH_HINT] = "_STORE_ATTR_WITH_HINT", + [_STORE_ATTR_WITH_HINT_r20] = "_STORE_ATTR_WITH_HINT_r20", [_STORE_DEREF] = "_STORE_DEREF", + [_STORE_DEREF_r10] = "_STORE_DEREF_r10", [_STORE_FAST] = "_STORE_FAST", + [_STORE_FAST_r10] = "_STORE_FAST_r10", [_STORE_FAST_0] = "_STORE_FAST_0", + [_STORE_FAST_0_r10] = "_STORE_FAST_0_r10", [_STORE_FAST_1] = "_STORE_FAST_1", + [_STORE_FAST_1_r10] = "_STORE_FAST_1_r10", [_STORE_FAST_2] = "_STORE_FAST_2", + [_STORE_FAST_2_r10] = "_STORE_FAST_2_r10", [_STORE_FAST_3] = "_STORE_FAST_3", + [_STORE_FAST_3_r10] = "_STORE_FAST_3_r10", [_STORE_FAST_4] = "_STORE_FAST_4", + [_STORE_FAST_4_r10] = "_STORE_FAST_4_r10", [_STORE_FAST_5] = "_STORE_FAST_5", + [_STORE_FAST_5_r10] = "_STORE_FAST_5_r10", [_STORE_FAST_6] = "_STORE_FAST_6", + [_STORE_FAST_6_r10] = "_STORE_FAST_6_r10", [_STORE_FAST_7] = "_STORE_FAST_7", - [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", - [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", + [_STORE_FAST_7_r10] = "_STORE_FAST_7_r10", [_STORE_GLOBAL] = "_STORE_GLOBAL", + [_STORE_GLOBAL_r10] = "_STORE_GLOBAL_r10", [_STORE_NAME] = "_STORE_NAME", + [_STORE_NAME_r10] = "_STORE_NAME_r10", [_STORE_SLICE] = "_STORE_SLICE", + [_STORE_SLICE_r30] = "_STORE_SLICE_r30", [_STORE_SUBSCR] = "_STORE_SUBSCR", + [_STORE_SUBSCR_r30] = "_STORE_SUBSCR_r30", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", + [_STORE_SUBSCR_DICT_r30] = "_STORE_SUBSCR_DICT_r30", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", + [_STORE_SUBSCR_LIST_INT_r30] = "_STORE_SUBSCR_LIST_INT_r30", [_SWAP] = "_SWAP", + [_SWAP_r11] = "_SWAP_r11", [_SWAP_2] = "_SWAP_2", + [_SWAP_2_r22] = "_SWAP_2_r22", + [_SWAP_2_r00] = "_SWAP_2_r00", + [_SWAP_2_r11] = "_SWAP_2_r11", + [_SWAP_2_r33] = "_SWAP_2_r33", [_SWAP_3] = "_SWAP_3", + [_SWAP_3_r33] = "_SWAP_3_r33", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", + [_TIER2_RESUME_CHECK_r00] = "_TIER2_RESUME_CHECK_r00", + [_TIER2_RESUME_CHECK_r11] = "_TIER2_RESUME_CHECK_r11", + [_TIER2_RESUME_CHECK_r22] = "_TIER2_RESUME_CHECK_r22", + [_TIER2_RESUME_CHECK_r33] = "_TIER2_RESUME_CHECK_r33", [_TO_BOOL] = "_TO_BOOL", + [_TO_BOOL_r11] = "_TO_BOOL_r11", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", + [_TO_BOOL_BOOL_r11] = "_TO_BOOL_BOOL_r11", + [_TO_BOOL_BOOL_r00] = "_TO_BOOL_BOOL_r00", + [_TO_BOOL_BOOL_r22] = "_TO_BOOL_BOOL_r22", + [_TO_BOOL_BOOL_r33] = "_TO_BOOL_BOOL_r33", [_TO_BOOL_INT] = "_TO_BOOL_INT", + [_TO_BOOL_INT_r11] = "_TO_BOOL_INT_r11", [_TO_BOOL_LIST] = "_TO_BOOL_LIST", + [_TO_BOOL_LIST_r11] = "_TO_BOOL_LIST_r11", [_TO_BOOL_NONE] = "_TO_BOOL_NONE", + [_TO_BOOL_NONE_r11] = "_TO_BOOL_NONE_r11", + [_TO_BOOL_NONE_r00] = "_TO_BOOL_NONE_r00", + [_TO_BOOL_NONE_r22] = "_TO_BOOL_NONE_r22", + [_TO_BOOL_NONE_r33] = "_TO_BOOL_NONE_r33", [_TO_BOOL_STR] = "_TO_BOOL_STR", + [_TO_BOOL_STR_r11] = "_TO_BOOL_STR_r11", [_UNARY_INVERT] = "_UNARY_INVERT", + [_UNARY_INVERT_r11] = "_UNARY_INVERT_r11", [_UNARY_NEGATIVE] = "_UNARY_NEGATIVE", + [_UNARY_NEGATIVE_r11] = "_UNARY_NEGATIVE_r11", [_UNARY_NOT] = "_UNARY_NOT", + [_UNARY_NOT_r11] = "_UNARY_NOT_r11", + [_UNARY_NOT_r00] = "_UNARY_NOT_r00", + [_UNARY_NOT_r22] = "_UNARY_NOT_r22", + [_UNARY_NOT_r33] = "_UNARY_NOT_r33", [_UNPACK_EX] = "_UNPACK_EX", + [_UNPACK_EX_r10] = "_UNPACK_EX_r10", [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", + [_UNPACK_SEQUENCE_r10] = "_UNPACK_SEQUENCE_r10", [_UNPACK_SEQUENCE_LIST] = "_UNPACK_SEQUENCE_LIST", + [_UNPACK_SEQUENCE_LIST_r10] = "_UNPACK_SEQUENCE_LIST_r10", [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", + [_UNPACK_SEQUENCE_TUPLE_r10] = "_UNPACK_SEQUENCE_TUPLE_r10", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", + [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_TWO_TUPLE_r12", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_WITH_EXCEPT_START_r33] = "_WITH_EXCEPT_START_r33", [_YIELD_VALUE] = "_YIELD_VALUE", + [_YIELD_VALUE_r11] = "_YIELD_VALUE_r11", }; int _PyUop_num_popped(int opcode, int oparg) { @@ -721,10 +4729,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_FAST_AND_CLEAR: return 0; - case _LOAD_FAST_LOAD_FAST: - return 0; - case _LOAD_FAST_BORROW_LOAD_FAST_BORROW: - return 0; case _LOAD_CONST: return 0; case _LOAD_SMALL_INT_0: @@ -755,10 +4759,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _STORE_FAST: return 1; - case _STORE_FAST_LOAD_FAST: - return 1; - case _STORE_FAST_STORE_FAST: - return 2; case _POP_TOP: return 1; case _POP_TOP_NOP: @@ -1307,6 +5307,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _TIER2_RESUME_CHECK: return 0; + case _SPILL_OR_RELOAD: + return 0; case _COLD_EXIT: return 0; default: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-20-16-03-59.gh-issue-135379.eDg89T.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-20-16-03-59.gh-issue-135379.eDg89T.rst new file mode 100644 index 00000000000000..7060b9b004cc25 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-20-16-03-59.gh-issue-135379.eDg89T.rst @@ -0,0 +1,6 @@ +Implement a limited form of register allocation known as "top of stack +caching" in the JIT. It works by keeping 0-3 of the top items in the stack +in registers. The code generator generates multiple versions of those uops +that do not escape and are relatively small. During JIT compilation, the +copy that produces the least memory traffic is selected, spilling or +reloading values when needed. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7f89c312b9a815..65bbad114353ec 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2332,20 +2332,22 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION_FT */ } - op(_LOAD_ATTR, (owner -- attr[1], self_or_null[oparg&1])) { + op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - *attr = PyStackRef_NULL; - int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); + _PyCStackRef method; + _PyThreadState_PushCStackRef(tstate, &method); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, &method.ref); if (is_meth) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(!PyStackRef_IsNull(*attr)); // No errors on this branch + assert(!PyStackRef_IsNull(method.ref)); // No errors on this branch self_or_null[0] = owner; // Transfer ownership DEAD(owner); + attr = _PyThreadState_PopCStackRefSteal(tstate, &method); } else { /* meth is not an unbound method (but a regular attr, or @@ -2355,8 +2357,9 @@ dummy_func( meth | NULL | arg1 | ... | argN */ PyStackRef_CLOSE(owner); - ERROR_IF(PyStackRef_IsNull(*attr)); self_or_null[0] = PyStackRef_NULL; + attr = _PyThreadState_PopCStackRefSteal(tstate, &method); + ERROR_IF(PyStackRef_IsNull(attr)); } } else { @@ -2364,7 +2367,7 @@ dummy_func( PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); PyStackRef_CLOSE(owner); ERROR_IF(attr_o == NULL); - *attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); } } @@ -2969,8 +2972,6 @@ dummy_func( else { this_instr[1].counter = initial_jump_backoff_counter(); assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; TIER1_TO_TIER2(executor); } } @@ -3035,8 +3036,6 @@ dummy_func( } DISPATCH_GOTO(); } - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -5354,6 +5353,9 @@ dummy_func( tier2 op(_START_EXECUTOR, (executor/4 --)) { #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); +#ifdef Py_DEBUG + assert(STACK_LEVEL() + current_cached_values == oparg); +#endif #endif assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); tstate->current_executor = (PyObject *)executor; @@ -5379,12 +5381,16 @@ dummy_func( } tier2 op(_DEOPT, (--)) { + SYNC_SP(); GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + Py_UNREACHABLE(); } tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { + SYNC_SP(); int err = _Py_HandlePending(tstate); GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + Py_UNREACHABLE(); } tier2 op(_ERROR_POP_N, (target/2 --)) { @@ -5406,6 +5412,9 @@ dummy_func( assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } + tier2 op(_SPILL_OR_RELOAD, (--)) { + } + tier2 op(_COLD_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); @@ -5413,9 +5422,11 @@ dummy_func( _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); + SYNC_SP(); GOTO_TIER_ONE(target); + Py_UNREACHABLE(); } - _PyExecutorObject *executor; + _PyExecutorObject *executor = NULL; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; @@ -5428,7 +5439,9 @@ dummy_func( int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); + SYNC_SP(); GOTO_TIER_ONE(optimized < 0 ? NULL : target); + Py_UNREACHABLE(); } exit->temperature = initial_temperature_backoff_counter(); } diff --git a/Python/ceval.c b/Python/ceval.c index 578c5d2a8b1420..26e4c58532e37a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -164,6 +164,16 @@ dump_item(_PyStackRef item) printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item)); return; } + if (PyStackRef_IsWrapped(item)) { + void *ptr = PyStackRef_Unwrap(item); + printf("Wrapped(pointer %p)", ptr); + return; + } + if (PyStackRef_IsError(item)) { + printf("ERROR"); + return; + } + assert(PyStackRef_IsValid(item)); PyObject *obj = PyStackRef_AsPyObjectBorrow(item); if (obj == NULL) { printf(""); @@ -201,11 +211,23 @@ dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) } printf("]\n"); } - fflush(stdout); PyErr_SetRaisedException(exc); _PyFrame_GetStackPointer(frame); } +#if defined(_Py_TIER2) && !defined(_Py_JIT) +static void +dump_cache_item(_PyStackRef cache, int position, int depth) +{ + if (position < depth) { + dump_item(cache); + } + else { + printf("---"); + } +} +#endif + static void lltrace_instruction(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, @@ -1153,10 +1175,19 @@ _PyTier2Interpreter( ) { const _PyUOpInstruction *next_uop; int oparg; -tier2_start: + /* Set up "jit" state after entry from tier 1. + * This mimics what the jit trampoline function does. */ + tstate->jit_exit = NULL; + _PyStackRef _tos_cache0 = PyStackRef_ZERO_BITS; + _PyStackRef _tos_cache1 = PyStackRef_ZERO_BITS; + _PyStackRef _tos_cache2 = PyStackRef_ZERO_BITS; + int current_cached_values = 0; + +tier2_start: next_uop = current_executor->trace; - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); + assert(next_uop->opcode == _START_EXECUTOR_r00 + current_cached_values || + next_uop->opcode == _COLD_EXIT_r00 + current_cached_values); #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 @@ -1180,14 +1211,25 @@ _PyTier2Interpreter( uint64_t trace_uop_execution_counter = 0; #endif - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); +#ifdef Py_DEBUG + assert(next_uop->opcode == _START_EXECUTOR_r00 + current_cached_values || + next_uop->opcode == _COLD_EXIT_r00 + current_cached_values); +#endif + tier2_dispatch: for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG if (frame->lltrace >= 3) { dump_stack(frame, stack_pointer); - if (next_uop->opcode == _START_EXECUTOR) { + printf(" cache=["); + dump_cache_item(_tos_cache0, 0, current_cached_values); + printf(", "); + dump_cache_item(_tos_cache1, 1, current_cached_values); + printf(", "); + dump_cache_item(_tos_cache2, 2, current_cached_values); + printf("]\n"); + if (next_uop->opcode == _START_EXECUTOR_r00) { printf("%4d uop: ", 0); } else { @@ -1195,6 +1237,7 @@ _PyTier2Interpreter( } _PyUOpPrint(next_uop); printf("\n"); + fflush(stdout); } #endif next_uop++; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 64ca7716fdbdee..4d5f341b2af2fc 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -209,6 +209,13 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define WITHIN_STACK_BOUNDS() \ (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) +#if defined(Py_DEBUG) && !defined(_Py_JIT) +#define WITHIN_STACK_BOUNDS_WITH_CACHE() \ + (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL() + current_cached_values) <= STACK_SIZE())) +#else +#define WITHIN_STACK_BOUNDS_WITH_CACHE WITHIN_STACK_BOUNDS +#endif + /* Data access macros */ #define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts) #define FRAME_CO_NAMES (_PyFrame_GetCode(frame)->co_names) @@ -351,6 +358,8 @@ _PyFrame_SetStackPointer(frame, stack_pointer) /* Tier-switching macros. */ +/* It is always safe to read stack_pointer[-1] and stack_pointer[-2] + * due to stack layout, even if the value is meaningless */ #define TIER1_TO_TIER2(EXECUTOR) \ do { \ OPT_STAT_INC(traces_executed); \ @@ -368,7 +377,7 @@ do { \ do { \ OPT_STAT_INC(traces_executed); \ current_executor = (EXECUTOR); \ - goto tier2_start; \ + goto tier2_start; \ } while (0) #define GOTO_TIER_ONE(TARGET) \ @@ -404,6 +413,14 @@ do { \ #define CONVERSION_FAILED(NAME) ((NAME) == NULL) +#if defined(Py_DEBUG) && !defined(_Py_JIT) +#define SET_CURRENT_CACHED_VALUES(N) current_cached_values = (N) +#define CHECK_CURRENT_CACHED_VALUES(N) assert(current_cached_values == (N)) +#else +#define SET_CURRENT_CACHED_VALUES(N) ((void)0) +#define CHECK_CURRENT_CACHED_VALUES(N) ((void)0) +#endif + static inline int check_periodics(PyThreadState *tstate) { _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3dcb2decc43737..96c70334f48b55 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8,32 +8,88 @@ #endif #define TIER_TWO 2 - case _NOP: { + case _NOP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_PERIODIC: { + case _NOP_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _NOP_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _NOP_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_PERIODIC_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } /* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 because it is replaced */ - case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); oparg = CURRENT_OPARG(); if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } @@ -41,10 +97,118 @@ /* _LOAD_BYTECODE is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _RESUME_CHECK: { + case _RESUME_CHECK_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _RESUME_CHECK_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _RESUME_CHECK_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _RESUME_CHECK_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -54,21 +218,30 @@ assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } /* _MONITOR_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _LOAD_FAST_CHECK: { + case _LOAD_FAST_CHECK_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; oparg = CURRENT_OPARG(); _PyStackRef value_s = GETLOCAL(oparg); @@ -79,4489 +252,9104 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_0: { + case _LOAD_FAST_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_1: { + case _LOAD_FAST_0_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 1; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_2: { + case _LOAD_FAST_0_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 2; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_3: { + case _LOAD_FAST_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 3; + oparg = 1; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_4: { + case _LOAD_FAST_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 4; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_5: { + case _LOAD_FAST_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 5; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_6: { + case _LOAD_FAST_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 6; + oparg = 2; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_7: { + case _LOAD_FAST_2_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 7; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST: { + case _LOAD_FAST_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = CURRENT_OPARG(); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; + assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_BORROW_0: { + case _LOAD_FAST_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 0; + oparg = 3; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_BORROW_1: { + case _LOAD_FAST_3_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 1; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_BORROW_2: { + case _LOAD_FAST_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 2; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_BORROW_3: { + case _LOAD_FAST_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 3; + oparg = 4; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_BORROW_4: { + case _LOAD_FAST_4_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = 4; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_BORROW_5: { + case _LOAD_FAST_4_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 5; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 4; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_BORROW_6: { + case _LOAD_FAST_5_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 6; + oparg = 5; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_BORROW_7: { + case _LOAD_FAST_5_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 7; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_BORROW: { + case _LOAD_FAST_5_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = CURRENT_OPARG(); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 5; + assert(oparg == CURRENT_OPARG()); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FAST_AND_CLEAR: { + case _LOAD_FAST_6_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = CURRENT_OPARG(); - value = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_CONST: { + case _LOAD_FAST_6_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = CURRENT_OPARG(); - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_SMALL_INT_0: { + case _LOAD_FAST_6_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 0; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 6; assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_SMALL_INT_1: { + case _LOAD_FAST_7_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 1; + oparg = 7; assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_SMALL_INT_2: { + case _LOAD_FAST_7_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 2; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_SMALL_INT_3: { + case _LOAD_FAST_7_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 3; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 7; assert(oparg == CURRENT_OPARG()); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_SMALL_INT: { + case _LOAD_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; oparg = CURRENT_OPARG(); - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_FAST_0: { + case _LOAD_FAST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 0; - assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_FAST_1: { + case _LOAD_FAST_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 1; - assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_FAST_2: { + case _LOAD_FAST_BORROW_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 2; + oparg = 0; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_FAST_3: { + case _LOAD_FAST_BORROW_0_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 3; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_FAST_4: { + case _LOAD_FAST_BORROW_0_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 4; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_FAST_5: { + case _LOAD_FAST_BORROW_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 5; + oparg = 1; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_FAST_6: { + case _LOAD_FAST_BORROW_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 6; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_FAST_7: { + case _LOAD_FAST_BORROW_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = 7; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; assert(oparg == CURRENT_OPARG()); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_FAST: { + case _LOAD_FAST_BORROW_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = CURRENT_OPARG(); - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_TOP: { + case _LOAD_FAST_BORROW_2_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_TOP_NOP: { + case _LOAD_FAST_BORROW_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || - _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_TOP_INT: { + case _LOAD_FAST_BORROW_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_TOP_FLOAT: { + case _LOAD_FAST_BORROW_3_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_TOP_UNICODE: { + case _LOAD_FAST_BORROW_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); - PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_TWO: { - _PyStackRef tos; - _PyStackRef nos; - tos = stack_pointer[-1]; - nos = stack_pointer[-2]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(tos); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(nos); - stack_pointer = _PyFrame_GetStackPointer(frame); + case _LOAD_FAST_BORROW_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _PUSH_NULL: { - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_FAST_BORROW_4_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _END_FOR: { + case _LOAD_FAST_BORROW_4_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_ITER: { - _PyStackRef index_or_null; - _PyStackRef iter; - index_or_null = stack_pointer[-1]; - iter = stack_pointer[-2]; - (void)index_or_null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + case _LOAD_FAST_BORROW_5_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _END_SEND: { + case _LOAD_FAST_BORROW_5_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef receiver; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _UNARY_NEGATIVE: { + case _LOAD_FAST_BORROW_5_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _UNARY_NOT: { + case _LOAD_FAST_BORROW_6_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _TO_BOOL: { + case _LOAD_FAST_BORROW_6_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_ERROR(); - } - res = err ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _TO_BOOL_BOOL: { + case _LOAD_FAST_BORROW_6_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - if (!PyStackRef_BoolCheck(value)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _TO_BOOL_INT: { + case _LOAD_FAST_BORROW_7_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; - } - stack_pointer[-1] = res; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_NOS_LIST: { - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _LOAD_FAST_BORROW_7_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_TOS_LIST: { - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _LOAD_FAST_BORROW_7_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_TOS_SLICE: { - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySlice_Check(o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _LOAD_FAST_BORROW_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _TO_BOOL_LIST: { + case _LOAD_FAST_BORROW_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyList_CheckExact(value_o)); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = res; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _TO_BOOL_NONE: { + case _LOAD_FAST_BORROW_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - if (!PyStackRef_IsNone(value)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_NOS_UNICODE: { - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _LOAD_FAST_AND_CLEAR_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_TOS_UNICODE: { + case _LOAD_FAST_AND_CLEAR_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _TO_BOOL_STR: { + case _LOAD_FAST_AND_CLEAR_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - assert(Py_SIZE(value_o)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; - } - stack_pointer[-1] = res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _REPLACE_WITH_TRUE: { + case _LOAD_CONST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _UNARY_INVERT: { + case _LOAD_CONST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_NOS_INT: { - _PyStackRef left; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _LOAD_CONST_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_TOS_INT: { + case _LOAD_SMALL_INT_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_NOS_OVERFLOWED: { - _PyStackRef left; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(Py_TYPE(left_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)left_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _LOAD_SMALL_INT_0_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_TOS_OVERFLOWED: { + case _LOAD_SMALL_INT_0_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(Py_TYPE(value_o) == &PyLong_Type); - if (!_PyLong_IsCompact((PyLongObject *)value_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_MULTIPLY_INT: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_SMALL_INT_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_ADD_INT: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_SMALL_INT_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_SUBTRACT_INT: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_SMALL_INT_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_NOS_FLOAT: { - _PyStackRef left; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _LOAD_SMALL_INT_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_TOS_FLOAT: { + case _LOAD_SMALL_INT_2_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_MULTIPLY_FLOAT: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_SMALL_INT_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_ADD_FLOAT: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_SMALL_INT_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_SUBTRACT_FLOAT: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_SMALL_INT_3_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_SMALL_INT_3_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_SMALL_INT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_SMALL_INT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_ADD_UNICODE: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _LOAD_SMALL_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_INPLACE_ADD_UNICODE: { - _PyStackRef right; - _PyStackRef left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = CURRENT_OPERAND0(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - assert(PyUnicode_CheckExact(left_o)); - if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyObject *right_o = PyStackRef_AsPyObjectSteal(right); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + case _STORE_FAST_0_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; _PyFrame_SetStackPointer(frame, stack_pointer); - PyUnicode_Append(&temp, right_o); + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - *target_local = PyStackRef_FromPyObjectSteal(temp); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _STORE_FAST_1_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(right_o); + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*target_local)) { - JUMP_TO_ERROR(); - } - #if TIER_ONE - - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - #endif + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_BINARY_OP_EXTEND: { - _PyStackRef right; - _PyStackRef left; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - assert(d && d->guard); + case _STORE_FAST_2_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; _PyFrame_SetStackPointer(frame, stack_pointer); - int res = d->guard(left_o, right_o); + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - if (!res) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_EXTEND: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - STAT_INC(BINARY_OP, hit); + case _STORE_FAST_3_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = d->action(left_o, right_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_SLICE: { - _PyStackRef stop; - _PyStackRef start; - _PyStackRef container; - _PyStackRef res; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; + case _STORE_FAST_4_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - if (slice == NULL) { - res_o = NULL; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _STORE_FAST_5_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(container); + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_SLICE: { - _PyStackRef stop; - _PyStackRef start; - _PyStackRef container; - _PyStackRef v; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; + case _STORE_FAST_6_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - int err; - if (slice == NULL) { - err = 1; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _STORE_FAST_7_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = container; - container = PyStackRef_NULL; - stack_pointer[-3] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-4] = v; - PyStackRef_CLOSE(tmp); + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - JUMP_TO_ERROR(); - } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_SUBSCR_LIST_INT: { - _PyStackRef sub_st; - _PyStackRef list_st; - _PyStackRef res; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED + case _STORE_FAST_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + value = _stack_item_0; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - res = PyStackRef_FromPyObjectSteal(res_o); - #else - if (index >= PyList_GET_SIZE(list)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(BINARY_OP, hit); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = list_st; - list_st = res; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_SUBSCR_LIST_SLICE: { - _PyStackRef sub_st; - _PyStackRef list_st; - _PyStackRef res; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PySlice_Check(sub)); - assert(PyList_CheckExact(list)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = list_st; - list_st = PyStackRef_NULL; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - JUMP_TO_ERROR(); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + case _POP_TOP_NOP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_SUBSCR_STR_INT: { - _PyStackRef sub_st; - _PyStackRef str_st; - _PyStackRef res; - sub_st = stack_pointer[-1]; - str_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - assert(PyLong_CheckExact(sub)); - assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; + case _POP_TOP_NOP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_NOP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || + _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_INT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_INT_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_FLOAT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_FLOAT_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_UNICODE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_UNICODE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TOP_UNICODE_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_TWO_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + nos = _stack_item_0; + stack_pointer[0] = nos; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str_st); + PyStackRef_CLOSE(tos); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectBorrow(res_o); - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(nos); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_NOS_TUPLE: { - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyTuple_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _PUSH_NULL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef res; + res = PyStackRef_NULL; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_TOS_TUPLE: { - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _PUSH_NULL_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + res = PyStackRef_NULL; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_SUBSCR_TUPLE_INT: { - _PyStackRef sub_st; - _PyStackRef tuple_st; + case _PUSH_NULL_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef res; - sub_st = stack_pointer[-1]; - tuple_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - res = PyStackRef_FromPyObjectNew(res_o); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + res = PyStackRef_NULL; + _tos_cache2 = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _END_FOR_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = tuple_st; - tuple_st = res; - stack_pointer[-1] = tuple_st; - PyStackRef_CLOSE(tmp); + PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_NOS_DICT: { - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _POP_ITER_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef index_or_null; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + index_or_null = _stack_item_1; + iter = _stack_item_0; + (void)index_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_TOS_DICT: { - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _END_SEND_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef receiver; + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + receiver = _stack_item_0; + val = value; + stack_pointer[0] = val; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = val; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_SUBSCR_DICT: { - _PyStackRef sub_st; - _PyStackRef dict_st; + case _UNARY_NEGATIVE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; _PyStackRef res; - sub_st = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = dict_st; - dict_st = PyStackRef_NULL; - stack_pointer[-2] = dict_st; - PyStackRef_CLOSE(tmp); + PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (rc <= 0) { + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_SUBSCR_CHECK_FUNC: { - _PyStackRef container; - _PyStackRef getitem; - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - if (getitem_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - getitem = PyStackRef_FromPyObjectNew(getitem_o); - STAT_INC(BINARY_OP, hit); - stack_pointer[0] = getitem; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + case _UNARY_NOT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP_SUBSCR_INIT_CALL: { - _PyStackRef getitem; - _PyStackRef sub; - _PyStackRef container; - _PyStackRef new_frame; - getitem = stack_pointer[-1]; - sub = stack_pointer[-2]; - container = stack_pointer[-3]; - _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - pushed_frame->localsplus[0] = container; - pushed_frame->localsplus[1] = sub; - frame->return_offset = 6 ; - new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-3] = new_frame; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + case _UNARY_NOT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[-1] = res; + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LIST_APPEND: { - _PyStackRef v; - _PyStackRef list; - oparg = CURRENT_OPARG(); - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _UNARY_NOT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _SET_ADD: { - _PyStackRef v; - _PyStackRef set; - oparg = CURRENT_OPARG(); - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _UNARY_NOT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_SUBSCR: { - _PyStackRef sub; - _PyStackRef container; - _PyStackRef v; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - v = stack_pointer[-3]; + case _TO_BOOL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-3] = v; - PyStackRef_CLOSE(tmp); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + res = err ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_SUBSCR_LIST_INT: { - _PyStackRef sub_st; - _PyStackRef list_st; + case _TO_BOOL_BOOL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - sub_st = stack_pointer[-1]; - list_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + if (!PyStackRef_BoolCheck(value)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (!LOCK_OBJECT(list)) { + STAT_INC(TO_BOOL, hit); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _TO_BOOL_BOOL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + if (!PyStackRef_BoolCheck(value)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], - PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + STAT_INC(TO_BOOL, hit); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_SUBSCR_DICT: { - _PyStackRef sub; - _PyStackRef dict_st; + case _TO_BOOL_BOOL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - sub = stack_pointer[-1]; - dict_st = stack_pointer[-2]; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(STORE_SUBSCR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_ERROR(); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } + STAT_INC(TO_BOOL, hit); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _DELETE_SUBSCR: { - _PyStackRef sub; - _PyStackRef container; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), - PyStackRef_AsPyObjectBorrow(sub)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - JUMP_TO_ERROR(); + case _TO_BOOL_BOOL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + if (!PyStackRef_BoolCheck(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } + STAT_INC(TO_BOOL, hit); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_INTRINSIC_1: { + case _TO_BOOL_INT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; _PyStackRef res; - oparg = CURRENT_OPARG(); - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_ERROR(); + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + } + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_INTRINSIC_2: { - _PyStackRef value1_st; - _PyStackRef value2_st; - _PyStackRef res; - oparg = CURRENT_OPARG(); - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - _PyStackRef tmp = value1_st; - value1_st = PyStackRef_NULL; - stack_pointer[-1] = value1_st; - PyStackRef_CLOSE(tmp); - tmp = value2_st; - value2_st = PyStackRef_NULL; - stack_pointer[-2] = value2_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - JUMP_TO_ERROR(); + case _GUARD_NOS_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _RETURN_VALUE: { - _PyStackRef retval; - _PyStackRef res; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + case _GUARD_NOS_LIST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GET_AITER: { - _PyStackRef obj; - _PyStackRef iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; + case _GUARD_NOS_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - if (getter == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_ERROR(); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - if (Py_TYPE(iter_o)->tp_as_async == NULL || - Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter_o)->tp_name); - Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_LIST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GET_ANEXT: { - _PyStackRef aiter; - _PyStackRef awaitable; - aiter = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - JUMP_TO_ERROR(); + case _GUARD_TOS_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GET_AWAITABLE: { - _PyStackRef iterable; - _PyStackRef iter; - oparg = CURRENT_OPARG(); - iterable = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_ERROR(); + case _GUARD_TOS_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _GUARD_TOS_SLICE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } - case _SEND_GEN_FRAME: { - _PyStackRef v; - _PyStackRef receiver; - _PyStackRef gen_frame; - oparg = CURRENT_OPARG(); - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + case _GUARD_TOS_SLICE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (gen->gi_frame_state >= FRAME_EXECUTING) { + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_SLICE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - STAT_INC(SEND, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); - pushed_frame->previous = frame; - gen_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-1] = gen_frame; + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _YIELD_VALUE: { - _PyStackRef retval; + case _GUARD_TOS_SLICE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _TO_BOOL_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = CURRENT_OPARG(); - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = PyStackRef_MakeHeapSafe(temp); - LLTRACE_RESUME_FRAME(); + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _POP_EXCEPT: { - _PyStackRef exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, - PyStackRef_IsNone(exc_value) - ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_COMMON_CONSTANT: { + case _TO_BOOL_NONE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - oparg = CURRENT_OPARG(); - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_BUILD_CLASS: { - _PyStackRef bc; - PyObject *bc_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_ERROR(); - } - if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + case _TO_BOOL_NONE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[-1] = res; + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_NAME: { - _PyStackRef v; - oparg = CURRENT_OPARG(); - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); - } - if (PyDict_CheckExact(ns)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + case _TO_BOOL_NONE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _TO_BOOL_NONE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + if (!PyStackRef_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_ERROR(); + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_UNICODE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _DELETE_NAME: { - oparg = CURRENT_OPARG(); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + case _GUARD_NOS_UNICODE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_UNICODE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _UNPACK_SEQUENCE: { - _PyStackRef seq; - _PyStackRef *top; - oparg = CURRENT_OPARG(); - seq = stack_pointer[-1]; - top = &stack_pointer[-1 + oparg]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - JUMP_TO_ERROR(); + case _GUARD_NOS_UNICODE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } - stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _UNPACK_SEQUENCE_TWO_TUPLE: { - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - oparg = CURRENT_OPARG(); - seq = stack_pointer[-1]; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != 2) { + case _GUARD_TOS_UNICODE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _UNPACK_SEQUENCE_TUPLE: { - _PyStackRef seq; - _PyStackRef *values; - oparg = CURRENT_OPARG(); - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != oparg) { + case _GUARD_TOS_UNICODE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_UNICODE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _UNPACK_SEQUENCE_LIST: { - _PyStackRef seq; - _PyStackRef *values; - oparg = CURRENT_OPARG(); - seq = stack_pointer[-1]; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyList_CheckExact(seq_o)); - if (!LOCK_OBJECT(seq_o)) { + case _GUARD_TOS_UNICODE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _TO_BOOL_STR_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); + else { + assert(Py_SIZE(value_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; } - UNLOCK_OBJECT(seq_o); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _REPLACE_WITH_TRUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); + PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _UNPACK_EX: { - _PyStackRef seq; - _PyStackRef *top; - oparg = CURRENT_OPARG(); - seq = stack_pointer[-1]; - top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + case _UNARY_INVERT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq_o); + PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_ATTR: { - _PyStackRef owner; - _PyStackRef v; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = owner; - owner = PyStackRef_NULL; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-2] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - JUMP_TO_ERROR(); + case _GUARD_NOS_INT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _DELETE_ATTR: { - _PyStackRef owner; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_ERROR(); + case _GUARD_NOS_INT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_GLOBAL: { - _PyStackRef v; - oparg = CURRENT_OPARG(); - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_ERROR(); + case _GUARD_NOS_INT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _DELETE_GLOBAL: { - oparg = CURRENT_OPARG(); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_ERROR(); - } - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + case _GUARD_NOS_INT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_LOCALS: { - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + case _GUARD_TOS_INT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ - - case _LOAD_NAME: { - _PyStackRef v; - oparg = CURRENT_OPARG(); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - JUMP_TO_ERROR(); + case _GUARD_TOS_INT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_GLOBAL: { - _PyStackRef *res; - oparg = CURRENT_OPARG(); - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) { - JUMP_TO_ERROR(); + case _GUARD_TOS_INT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _PUSH_NULL_CONDITIONAL: { - _PyStackRef *null; - oparg = CURRENT_OPARG(); - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; + case _GUARD_TOS_INT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_GLOBALS_VERSION: { - uint16_t version = (uint16_t)CURRENT_OPERAND0(); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + case _GUARD_NOS_OVERFLOWED_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - assert(DK_IS_UNICODE(keys)); + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_GLOBAL_MODULE: { - _PyStackRef res; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); - uint16_t index = (uint16_t)CURRENT_OPERAND1(); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { + case _GUARD_NOS_OVERFLOWED_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_OVERFLOWED_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - assert(DK_IS_UNICODE(keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_OVERFLOWED_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(Py_TYPE(left_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)left_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_OVERFLOWED_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_GLOBAL_BUILTINS: { - _PyStackRef res; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); - uint16_t index = (uint16_t)CURRENT_OPERAND1(); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) { + case _GUARD_TOS_OVERFLOWED_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_OVERFLOWED_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - assert(DK_IS_UNICODE(keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_OVERFLOWED_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(Py_TYPE(value_o) == &PyLong_Type); + if (!_PyLong_IsCompact((PyLongObject *)value_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _BINARY_OP_MULTIPLY_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _DELETE_FAST: { - oparg = CURRENT_OPARG(); - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + case _BINARY_OP_ADD_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _MAKE_CELL: { - oparg = CURRENT_OPARG(); - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - JUMP_TO_ERROR(); + case _BINARY_OP_SUBTRACT_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _DELETE_DEREF: { - oparg = CURRENT_OPARG(); - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + case _GUARD_NOS_FLOAT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache1 = _stack_item_1; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_FROM_DICT_OR_DEREF: { - _PyStackRef class_dict_st; - _PyStackRef value; - oparg = CURRENT_OPARG(); - class_dict_st = stack_pointer[-1]; - PyObject *value_o; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_ERROR(); - } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); - } + case _GUARD_NOS_FLOAT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_DEREF: { - _PyStackRef value; - oparg = CURRENT_OPARG(); - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(value)) { - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + case _GUARD_NOS_FLOAT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + left = stack_pointer[-1]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_DEREF: { - _PyStackRef v; - oparg = CURRENT_OPARG(); - v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + case _GUARD_NOS_FLOAT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + left = _stack_item_1; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _COPY_FREE_VARS: { - oparg = CURRENT_OPARG(); - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + case _GUARD_TOS_FLOAT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BUILD_STRING: { - _PyStackRef *pieces; - _PyStackRef str; - oparg = CURRENT_OPARG(); - pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - if (str_o == NULL) { - JUMP_TO_ERROR(); + case _GUARD_TOS_FLOAT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BUILD_INTERPOLATION: { - _PyStackRef *format; - _PyStackRef str; + case _GUARD_TOS_FLOAT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - _PyStackRef interpolation; - oparg = CURRENT_OPARG(); - format = &stack_pointer[-(oparg & 1)]; - str = stack_pointer[-1 - (oparg & 1)]; - value = stack_pointer[-2 - (oparg & 1)]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); - int conversion = oparg >> 2; - PyObject *format_o; - if (oparg & 1) { - format_o = PyStackRef_AsPyObjectBorrow(format[0]); - } - else { - format_o = &_Py_STR(empty); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (oparg & 1) { - stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - stack_pointer += -(oparg & 1); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (interpolation_o == NULL) { - JUMP_TO_ERROR(); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_FLOAT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } - interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); - stack_pointer[0] = interpolation; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BUILD_TEMPLATE: { - _PyStackRef interpolations; - _PyStackRef strings; - _PyStackRef template; - interpolations = stack_pointer[-1]; - strings = stack_pointer[-2]; - PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); - PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (template_o == NULL) { + case _BINARY_OP_MULTIPLY_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - template = PyStackRef_FromPyObjectSteal(template_o); - stack_pointer[0] = template; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BUILD_TUPLE: { - _PyStackRef *values; - _PyStackRef tup; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); - if (tup_o == NULL) { + case _BINARY_OP_ADD_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - tup = PyStackRef_FromPyObjectStealMortal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BUILD_LIST: { - _PyStackRef *values; - _PyStackRef list; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (list_o == NULL) { + case _BINARY_OP_SUBTRACT_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - list = PyStackRef_FromPyObjectStealMortal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LIST_EXTEND: { - _PyStackRef iterable_st; - _PyStackRef list_st; - oparg = CURRENT_OPARG(); - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - assert(Py_IsNone(none_val)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _SET_UPDATE: { - _PyStackRef iterable; - _PyStackRef set; - oparg = CURRENT_OPARG(); - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BUILD_SET: { - _PyStackRef *values; - _PyStackRef set; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - int err = 0; - for (Py_ssize_t i = 0; i < oparg; i++) { - _PyStackRef value = values[i]; - values[i] = PyStackRef_NULL; - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - if (err) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); + if (PyStackRef_IsNull(res)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - set = PyStackRef_FromPyObjectStealMortal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BUILD_MAP: { - _PyStackRef *values; - _PyStackRef map; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - if (map_o == NULL) { + case _BINARY_OP_ADD_UNICODE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _SETUP_ANNOTATIONS: { - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + case _BINARY_OP_INPLACE_ADD_UNICODE_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } + STAT_INC(BINARY_OP, hit); + assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + PyUnicode_Append(&temp, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + *target_local = PyStackRef_FromPyObjectSteal(temp); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*target_local)) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) { - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - JUMP_TO_ERROR(); - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - } + #if TIER_ONE + + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _DICT_UPDATE: { - _PyStackRef update; - _PyStackRef dict; - oparg = CURRENT_OPARG(); - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + case _GUARD_BINARY_OP_EXTEND_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); + int res = d->guard(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + if (!res) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - stack_pointer += -1; + _tos_cache1 = right; + _tos_cache0 = left; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _BINARY_OP_EXTEND_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); + PyObject *res_o = d->action(left_o, right_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _DICT_MERGE: { - _PyStackRef update; - _PyStackRef dict; - _PyStackRef callable; - oparg = CURRENT_OPARG(); - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + case _BINARY_SLICE_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + stop = _stack_item_2; + start = _stack_item_1; + container = _stack_item_0; + stack_pointer[0] = container; + stack_pointer[1] = start; + stack_pointer[2] = stop; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; + PyObject *res_o; + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + Py_DECREF(slice); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); + stack_pointer += 2; } - stack_pointer += -1; + stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); + PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _MAP_ADD: { - _PyStackRef value; - _PyStackRef key; - _PyStackRef dict_st; - oparg = CURRENT_OPARG(); - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict_st = stack_pointer[-3 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); + case _STORE_SLICE_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef stop; + _PyStackRef start; + _PyStackRef container; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + stop = _stack_item_2; + start = _stack_item_1; + container = _stack_item_0; + v = stack_pointer[-1]; + stack_pointer[0] = container; + stack_pointer[1] = start; + stack_pointer[2] = stop; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { + int err; + if (slice == NULL) { + err = 1; + } + else { stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; } - stack_pointer += -2; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-4] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_SUPER_ATTR_ATTR: { - _PyStackRef self_st; - _PyStackRef class_st; - _PyStackRef global_super_st; - _PyStackRef attr_st; - oparg = CURRENT_OPARG(); - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) { + case _BINARY_OP_SUBSCR_LIST_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + list_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - if (!PyType_Check(class)) { + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + stack_pointer[0] = list_st; + stack_pointer[1] = sub_st; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer += 2; + #endif + STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; + _PyStackRef tmp = list_st; + list_st = res; + stack_pointer[-2] = list_st; PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; + tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - if (attr == NULL) { - JUMP_TO_ERROR(); - } - attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[0] = attr_st; - stack_pointer += 1; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_SUPER_ATTR_METHOD: { - _PyStackRef self_st; - _PyStackRef class_st; - _PyStackRef global_super_st; - _PyStackRef attr; - _PyStackRef self_or_null; - oparg = CURRENT_OPARG(); - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (!PyType_Check(class)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; + case _BINARY_OP_SUBSCR_LIST_SLICE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef sub_st; + _PyStackRef list_st; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + list_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PySlice_Check(sub)); + assert(PyList_CheckExact(list)); + stack_pointer[0] = list_st; + stack_pointer[1] = sub_st; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + PyObject *res_o = _PyList_SliceSubscript(list, sub); stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - JUMP_TO_ERROR(); - } - if (method_found) { - self_or_null = self_st; - } else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - stack_pointer += 1; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = global_super_st; - global_super_st = self_or_null; - stack_pointer[-2] = global_super_st; + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-1] = class_st; + tmp = list_st; + list_st = PyStackRef_NULL; + stack_pointer[-2] = list_st; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR: { - _PyStackRef owner; - _PyStackRef *attr; - _PyStackRef *self_or_null; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - attr = &stack_pointer[-1]; - self_or_null = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - *attr = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - assert(!PyStackRef_IsNull(*attr)); - self_or_null[0] = owner; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*attr)) { - JUMP_TO_ERROR(); - } - self_or_null[0] = PyStackRef_NULL; - stack_pointer += 1; - } + case _BINARY_OP_SUBSCR_STR_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef sub_st; + _PyStackRef str_st; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + str_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - JUMP_TO_ERROR(); - } - *attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer += 1; + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectBorrow(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_TYPE_VERSION: { - _PyStackRef owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + case _GUARD_NOS_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_TYPE_VERSION_AND_LOCK: { - _PyStackRef owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - if (!LOCK_OBJECT(owner_o)) { + case _GUARD_NOS_TUPLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_MANAGED_OBJECT_HAS_VALUES: { - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + case _GUARD_NOS_TUPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_INSTANCE_VALUE: { - _PyStackRef owner; - _PyStackRef attr; - owner = stack_pointer[-1]; - uint16_t offset = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { + case _GUARD_NOS_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); - if (!increfed) { - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_MODULE: { - _PyStackRef owner; - _PyStackRef attr; - owner = stack_pointer[-1]; - uint32_t dict_version = (uint32_t)CURRENT_OPERAND0(); - uint16_t index = (uint16_t)CURRENT_OPERAND1(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + case _GUARD_TOS_TUPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_TUPLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - if (attr_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_WITH_HINT: { - _PyStackRef owner; - _PyStackRef attr; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - uint16_t hint = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { + case _GUARD_TOS_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); - assert(PyDict_CheckExact((PyObject *)dict)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_TUPLE_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + tuple_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - #endif - PyObject *attr_o; - if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (dk->dk_kind != DICT_KEYS_UNICODE) { - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer[0] = tuple_st; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = tuple_st; + tuple_st = res; + stack_pointer[-1] = tuple_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; - if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); - if (attr_o == NULL) { - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - STAT_INC(LOAD_ATTR, hit); - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_SLOT: { - _PyStackRef owner; - _PyStackRef attr; - owner = stack_pointer[-1]; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { + case _GUARD_TOS_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_ATTR_CLASS: { - _PyStackRef owner; - owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { + case _GUARD_TOS_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_CLASS: { - _PyStackRef owner; - _PyStackRef attr; - owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); + case _BINARY_OP_SUBSCR_DICT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef sub_st; + _PyStackRef dict_st; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + dict_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + stack_pointer[0] = dict_st; + stack_pointer[1] = sub_st; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (rc <= 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_PROPERTY_FRAME: { - _PyStackRef owner; - _PyStackRef new_frame; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - PyObject *fget = (PyObject *)CURRENT_OPERAND0(); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + case _BINARY_OP_SUBSCR_CHECK_FUNC_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + container = _stack_item_0; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - if (code->co_kwonlyargcount) { + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - if (code->co_argcount != 1) { + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - STAT_INC(LOAD_ATTR, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - pushed_frame->localsplus[0] = owner; - new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-1] = new_frame; + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_OP, hit); + _tos_cache2 = getitem; + _tos_cache1 = _stack_item_1; + _tos_cache0 = container; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has too many cache entries */ - - case _GUARD_DORV_NO_DICT: { - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + case _BINARY_OP_SUBSCR_INIT_CALL_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef container; + _PyStackRef new_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + getitem = _stack_item_2; + sub = _stack_item_1; + container = _stack_item_0; + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6 ; + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LIST_APPEND_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef v; + _PyStackRef list; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + list = stack_pointer[-1 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_ATTR_INSTANCE_VALUE: { - _PyStackRef owner; - _PyStackRef value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint16_t offset = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); + case _SET_ADD_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef v; + _PyStackRef set; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + set = stack_pointer[-1 - (oparg-1)]; + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); } - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _STORE_SUBSCR_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef sub; + _PyStackRef container; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub = _stack_item_2; + container = _stack_item_1; + v = _stack_item_0; + stack_pointer[0] = v; + stack_pointer[1] = container; + stack_pointer[2] = sub; + stack_pointer += 3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_ATTR_WITH_HINT: { - _PyStackRef owner; + case _STORE_SUBSCR_LIST_INT_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef sub_st; + _PyStackRef list_st; _PyStackRef value; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint16_t hint = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub_st = _stack_item_2; + list_st = _stack_item_1; + value = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - if (!LOCK_OBJECT(dict)) { + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { - UNLOCK_OBJECT(dict); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(dict); - STAT_INC(STORE_ATTR, hit); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); + PyStackRef_CLOSE(list_st); + Py_DECREF(old_value); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _STORE_ATTR_SLOT: { - _PyStackRef owner; + case _STORE_SUBSCR_DICT_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef sub; + _PyStackRef dict_st; _PyStackRef value; - owner = stack_pointer[-1]; - value = stack_pointer[-2]; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!LOCK_OBJECT(owner_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub = _stack_item_2; + dict_st = _stack_item_1; + value = _stack_item_0; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + stack_pointer[0] = value; + stack_pointer[1] = dict_st; + stack_pointer[2] = sub; + stack_pointer += 3; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _COMPARE_OP: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); + case _DELETE_SUBSCR_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef sub; + _PyStackRef container; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub = _stack_item_1; + container = _stack_item_0; + stack_pointer[0] = container; + stack_pointer[1] = sub; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { + if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - if (oparg & 16) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) { - JUMP_TO_ERROR(); - } - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _COMPARE_OP_FLOAT: { - _PyStackRef right; - _PyStackRef left; + case _CALL_INTRINSIC_1_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = res; + value = _stack_item_0; + assert(oparg <= MAX_INTRINSIC_1); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _COMPARE_OP_INT: { - _PyStackRef right; - _PyStackRef left; + case _CALL_INTRINSIC_2_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value1_st; + _PyStackRef value2_st; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _COMPARE_OP_STR: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[-2] = res; - stack_pointer += -1; + value1_st = _stack_item_1; + value2_st = _stack_item_0; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + stack_pointer[0] = value2_st; + stack_pointer[1] = value1_st; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _IS_OP: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyStackRef tmp = value1_st; + value1_st = PyStackRef_NULL; + stack_pointer[-1] = value1_st; PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; + tmp = value2_st; + value2_st = PyStackRef_NULL; + stack_pointer[-2] = value2_st; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CONTAINS_OP: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + case _RETURN_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef retval; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + retval = _stack_item_0; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GET_AITER_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef obj; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + obj = _stack_item_0; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + stack_pointer[0] = obj; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; + stack_pointer[0] = obj; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _GUARD_TOS_ANY_SET: { - _PyStackRef tos; - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnySet_CheckExact(o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); } + iter = PyStackRef_FromPyObjectSteal(iter_o); + _tos_cache0 = iter; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CONTAINS_OP_SET: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyAnySet_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); + case _GET_ANEXT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef aiter; + _PyStackRef awaitable; + _PyStackRef _stack_item_0 = _tos_cache0; + aiter = _stack_item_0; + stack_pointer[0] = aiter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { + if (awaitable_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + _tos_cache1 = awaitable; + _tos_cache0 = aiter; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CONTAINS_OP_DICT: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; + case _GET_AWAITABLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyDict_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); + iterable = _stack_item_0; + stack_pointer[0] = iterable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + iter = PyStackRef_FromPyObjectSteal(iter_o); + _tos_cache0 = iter; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_EG_MATCH: { - _PyStackRef match_type_st; - _PyStackRef exc_value_st; - _PyStackRef rest; - _PyStackRef match; - match_type_st = stack_pointer[-1]; - exc_value_st = stack_pointer[-2]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); + /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + + case _SEND_GEN_FRAME_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + v = _stack_item_1; + receiver = _stack_item_0; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, - &match_o, &rest_o); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - JUMP_TO_ERROR(); - } - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) { - JUMP_TO_ERROR(); - } - if (!Py_IsNone(match_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[0] = rest; - stack_pointer[1] = match; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2 + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2 + oparg); + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache1 = gen_frame; + _tos_cache0 = receiver; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_EXC_MATCH: { - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_ERROR(); - } + case _YIELD_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef retval; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + retval = _stack_item_0; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_EXCEPT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef exc_value; + _PyStackRef _stack_item_0 = _tos_cache0; + exc_value = _stack_item_0; + _PyErr_StackItem *exc_info = tstate->exc_info; + stack_pointer[0] = exc_value; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(right); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer = _PyFrame_GetStackPointer(frame); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _IMPORT_NAME: { - _PyStackRef fromlist; - _PyStackRef level; - _PyStackRef res; + case _LOAD_COMMON_CONSTANT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; oparg = CURRENT_OPARG(); - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_COMMON_CONSTANT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_COMMON_CONSTANT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_BUILD_CLASS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef bc; + PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - _PyStackRef tmp = fromlist; - fromlist = PyStackRef_NULL; - stack_pointer[-1] = fromlist; - PyStackRef_CLOSE(tmp); - tmp = level; - level = PyStackRef_NULL; - stack_pointer[-2] = level; - PyStackRef_CLOSE(tmp); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + _tos_cache0 = bc; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _IMPORT_FROM: { - _PyStackRef from; - _PyStackRef res; + case _STORE_NAME_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - from = stack_pointer[-1]; + v = _stack_item_0; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (PyDict_CheckExact(ns)) { + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { + if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ - - /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ - - case _IS_NONE: { - _PyStackRef value; - _PyStackRef b; - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; + case _DELETE_NAME_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); } - else { - b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = b; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); } - stack_pointer[-1] = b; + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GET_LEN: { - _PyStackRef obj; - _PyStackRef len; - obj = stack_pointer[-1]; + case _UNPACK_SEQUENCE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef seq; + _PyStackRef *top; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + top = &stack_pointer[oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - JUMP_TO_ERROR(); - } - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) { + if (res == 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += oparg; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_TWO_TUPLE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + stack_pointer[0] = val1; + stack_pointer[1] = val0; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache1 = val0; + _tos_cache0 = val1; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_TUPLE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef seq; + _PyStackRef *values; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + values = &stack_pointer[0]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + stack_pointer += oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _UNPACK_SEQUENCE_LIST_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef seq; + _PyStackRef *values; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + values = &stack_pointer[0]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyList_CheckExact(seq_o)); + if (!LOCK_OBJECT(seq_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + stack_pointer += oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _UNPACK_EX_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef seq; + _PyStackRef *top; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + top = &stack_pointer[1 + (oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _STORE_ATTR_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + v = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + stack_pointer[0] = v; + stack_pointer[1] = owner; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _DELETE_ATTR_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + stack_pointer[0] = owner; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _STORE_GLOBAL_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _DELETE_GLOBAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_LOCALS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + locals = PyStackRef_FromPyObjectNew(l); + _tos_cache0 = locals; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ + + case _LOAD_NAME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef v; + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + v = PyStackRef_FromPyObjectSteal(v_o); + _tos_cache0 = v; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_GLOBAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef *res; + oparg = CURRENT_OPARG(); + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _PUSH_NULL_CONDITIONAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef *null; + oparg = CURRENT_OPARG(); + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_GLOBALS_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(DK_IS_UNICODE(keys)); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_GLOBALS_VERSION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(DK_IS_UNICODE(keys)); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_GLOBALS_VERSION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(DK_IS_UNICODE(keys)); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_GLOBALS_VERSION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + assert(DK_IS_UNICODE(keys)); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_GLOBAL_MODULE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef res; + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t index = (uint16_t)CURRENT_OPERAND1(); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_GLOBAL_BUILTINS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef res; + uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t index = (uint16_t)CURRENT_OPERAND1(); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _DELETE_FAST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + oparg = CURRENT_OPARG(); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MAKE_CELL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + oparg = CURRENT_OPARG(); + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _DELETE_DEREF_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + oparg = CURRENT_OPARG(); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_FROM_DICT_OR_DEREF_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef class_dict_st; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + class_dict_st = _stack_item_0; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + stack_pointer[0] = class_dict_st; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectSteal(value_o); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_DEREF_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + oparg = CURRENT_OPARG(); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = _PyCell_GetStackRef(cell); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(value)) { + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _STORE_DEREF_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _COPY_FREE_VARS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + oparg = CURRENT_OPARG(); + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _BUILD_STRING_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef *pieces; + _PyStackRef str; + oparg = CURRENT_OPARG(); + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + if (str_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + str = PyStackRef_FromPyObjectSteal(str_o); + _tos_cache0 = str; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _BUILD_INTERPOLATION_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef *format; + _PyStackRef str; + _PyStackRef value; + _PyStackRef interpolation; + oparg = CURRENT_OPARG(); + format = &stack_pointer[-(oparg & 1)]; + str = stack_pointer[-1 - (oparg & 1)]; + value = stack_pointer[-2 - (oparg & 1)]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); + int conversion = oparg >> 2; + PyObject *format_o; + if (oparg & 1) { + format_o = PyStackRef_AsPyObjectBorrow(format[0]); + } + else { + format_o = &_Py_STR(empty); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (oparg & 1) { + stack_pointer += -(oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(format[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer += -(oparg & 1); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (interpolation_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); + _tos_cache0 = interpolation; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _BUILD_TEMPLATE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef interpolations; + _PyStackRef strings; + _PyStackRef template; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + interpolations = _stack_item_1; + strings = _stack_item_0; + PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); + PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); + stack_pointer[0] = strings; + stack_pointer[1] = interpolations; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(interpolations); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(strings); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (template_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + template = PyStackRef_FromPyObjectSteal(template_o); + _tos_cache0 = template; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _BUILD_TUPLE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef *values; + _PyStackRef tup; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + tup = PyStackRef_FromPyObjectStealMortal(tup_o); + _tos_cache0 = tup; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _BUILD_LIST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef *values; + _PyStackRef list; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (list_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + list = PyStackRef_FromPyObjectStealMortal(list_o); + _tos_cache0 = list; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LIST_EXTEND_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef iterable_st; + _PyStackRef list_st; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + iterable_st = _stack_item_0; + list_st = stack_pointer[-1 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + stack_pointer[0] = iterable_st; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(Py_IsNone(none_val)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SET_UPDATE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef iterable; + _PyStackRef set; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + iterable = _stack_item_0; + set = stack_pointer[-1 - (oparg-1)]; + stack_pointer[0] = iterable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _BUILD_SET_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef *values; + _PyStackRef set; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + int err = 0; + for (Py_ssize_t i = 0; i < oparg; i++) { + _PyStackRef value = values[i]; + values[i] = PyStackRef_NULL; + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + if (err) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + set = PyStackRef_FromPyObjectStealMortal(set_o); + _tos_cache0 = set; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _BUILD_MAP_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef *values; + _PyStackRef map; + oparg = CURRENT_OPARG(); + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg*2; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg*2; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + if (map_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + map = PyStackRef_FromPyObjectStealMortal(map_o); + _tos_cache0 = map; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SETUP_ANNOTATIONS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _DICT_UPDATE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef update; + _PyStackRef dict; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + update = _stack_item_0; + dict = stack_pointer[-1 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + stack_pointer[0] = update; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _DICT_MERGE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef update; + _PyStackRef dict; + _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + update = _stack_item_0; + dict = stack_pointer[-1 - (oparg - 1)]; + callable = stack_pointer[-4 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + stack_pointer[0] = update; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MAP_ADD_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef key; + _PyStackRef dict_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + value = _stack_item_1; + key = _stack_item_0; + dict_st = stack_pointer[-1 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + stack_pointer[0] = key; + stack_pointer[1] = value; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_SUPER_ATTR_ATTR_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + self_st = _stack_item_2; + class_st = _stack_item_1; + global_super_st = _stack_item_0; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + stack_pointer[0] = global_super_st; + stack_pointer[1] = class_st; + stack_pointer[2] = self_st; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (attr == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + attr_st = PyStackRef_FromPyObjectSteal(attr); + _tos_cache0 = attr_st; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_SUPER_ATTR_METHOD_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef self_st; + _PyStackRef class_st; + _PyStackRef global_super_st; + _PyStackRef attr; + _PyStackRef self_or_null; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + self_st = _stack_item_2; + class_st = _stack_item_1; + global_super_st = _stack_item_0; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + stack_pointer[0] = global_super_st; + stack_pointer[1] = class_st; + stack_pointer[2] = self_st; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + attr = PyStackRef_FromPyObjectSteal(attr_o); + _tos_cache1 = self_or_null; + _tos_cache0 = attr; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *self_or_null; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + self_or_null = &stack_pointer[1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + _PyCStackRef method; + stack_pointer[0] = owner; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyThreadState_PushCStackRef(tstate, &method); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, &method.ref); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + assert(!PyStackRef_IsNull(method.ref)); + self_or_null[0] = owner; + attr = _PyThreadState_PopCStackRefSteal(tstate, &method); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null[0] = PyStackRef_NULL; + attr = _PyThreadState_PopCStackRefSteal(tstate, &method); + if (PyStackRef_IsNull(attr)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + stack_pointer += 1; + } + } + else { + stack_pointer[0] = owner; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer += 1; + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[-1] = attr; + stack_pointer += (oparg&1); + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_AND_LOCK_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_AND_LOCK_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_AND_LOCK_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TYPE_VERSION_AND_LOCK_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); + if (!increfed) { + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_MODULE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t dict_version = (uint32_t)CURRENT_OPERAND0(); + uint16_t index = (uint16_t)CURRENT_OPERAND1(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_WITH_HINT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + uint16_t hint = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); + assert(PyDict_CheckExact((PyObject *)dict)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + PyObject *attr_o; + if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (dk->dk_kind != DICT_KEYS_UNICODE) { + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; + if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); + if (attr_o == NULL) { + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_SLOT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[0] = owner; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_ATTR_CLASS_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_ATTR_CLASS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_ATTR_CLASS_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_ATTR_CLASS_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_CLASS_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + stack_pointer[0] = owner; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_PROPERTY_FRAME_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef new_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *fget = (PyObject *)CURRENT_OPERAND0(); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (code->co_kwonlyargcount) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (code->co_argcount != 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(LOAD_ATTR, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it has too many cache entries */ + + case _GUARD_DORV_NO_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_DORV_NO_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_DORV_NO_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_DORV_NO_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _STORE_ATTR_INSTANCE_VALUE_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + value = _stack_item_0; + uint16_t offset = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _STORE_ATTR_WITH_HINT_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + value = _stack_item_0; + uint16_t hint = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!LOCK_OBJECT(dict)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + stack_pointer[0] = value; + stack_pointer[1] = owner; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + STAT_INC(STORE_ATTR, hit); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _STORE_ATTR_SLOT_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + value = _stack_item_0; + uint16_t index = (uint16_t)CURRENT_OPERAND0(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _COMPARE_OP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (oparg & 16) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + } + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _COMPARE_OP_FLOAT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _COMPARE_OP_INT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _COMPARE_OP_STR_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _IS_OP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[0] = left; + stack_pointer[1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[0] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + b = res ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CONTAINS_OP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_ANY_SET_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + tos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = tos; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_ANY_SET_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_ANY_SET_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + tos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = tos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_TOS_ANY_SET_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef tos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + tos = _stack_item_2; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = tos; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CONTAINS_OP_SET_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnySet_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CONTAINS_OP_DICT_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyDict_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_EG_MATCH_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef match_type_st; + _PyStackRef exc_value_st; + _PyStackRef rest; + _PyStackRef match; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + match_type_st = _stack_item_1; + exc_value_st = _stack_item_0; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + stack_pointer[0] = exc_value_st; + stack_pointer[1] = match_type_st; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (!Py_IsNone(match_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + _tos_cache1 = match; + _tos_cache0 = rest; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_EXC_MATCH_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); + b = res ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = b; + _tos_cache0 = left; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _IMPORT_NAME_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef fromlist; + _PyStackRef level; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + fromlist = _stack_item_1; + level = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + stack_pointer[0] = level; + stack_pointer[1] = fromlist; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _IMPORT_FROM_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef from; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + from = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + stack_pointer[0] = from; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache1 = res; + _tos_cache0 = from; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ + + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ + + case _IS_NONE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef b; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + } + _tos_cache0 = b; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GET_LEN_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef obj; + _PyStackRef len; + _PyStackRef _stack_item_0 = _tos_cache0; + obj = _stack_item_0; + stack_pointer[0] = obj; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + len = PyStackRef_FromPyObjectSteal(len_o); + _tos_cache1 = len; + _tos_cache0 = obj; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MATCH_CLASS_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef names; + _PyStackRef type; + _PyStackRef subject; + _PyStackRef attrs; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + names = _stack_item_2; + type = _stack_item_1; + subject = _stack_item_0; + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + stack_pointer[0] = subject; + stack_pointer[1] = type; + stack_pointer[2] = names; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + _PyStackRef tmp = names; + names = PyStackRef_NULL; + stack_pointer[-1] = names; + PyStackRef_CLOSE(tmp); + tmp = type; + type = PyStackRef_NULL; + stack_pointer[-2] = type; + PyStackRef_CLOSE(tmp); + tmp = subject; + subject = PyStackRef_NULL; + stack_pointer[-3] = subject; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + attrs = PyStackRef_None; + } + _tos_cache0 = attrs; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MATCH_MAPPING_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + subject = _stack_item_0; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = subject; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MATCH_MAPPING_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MATCH_MAPPING_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + subject = _stack_item_1; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = subject; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MATCH_SEQUENCE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + subject = _stack_item_0; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache1 = res; + _tos_cache0 = subject; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MATCH_SEQUENCE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MATCH_SEQUENCE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef subject; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + subject = _stack_item_1; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = res; + _tos_cache1 = subject; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MATCH_KEYS_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef keys; + _PyStackRef subject; + _PyStackRef values_or_none; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + keys = _stack_item_1; + subject = _stack_item_0; + stack_pointer[0] = subject; + stack_pointer[1] = keys; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + _tos_cache2 = values_or_none; + _tos_cache1 = keys; + _tos_cache0 = subject; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GET_ITER_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef index_or_null; + _PyStackRef _stack_item_0 = _tos_cache0; + iterable = _stack_item_0; + #ifdef Py_STATS + stack_pointer[0] = iterable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_GatherStats_GetIter(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + #endif + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); + } + else { + stack_pointer[0] = iterable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + } + _tos_cache1 = index_or_null; + _tos_cache0 = iter; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GET_YIELD_FROM_ITER_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + iterable = _stack_item_0; + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + stack_pointer[0] = iterable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + iter = iterable; + } + else if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + stack_pointer[0] = iterable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = iterable; + iterable = iter; + stack_pointer[-1] = iterable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + } + _tos_cache0 = iter; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ + + case _FOR_ITER_TIER_TWO_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + stack_pointer[0] = iter; + stack_pointer[1] = null_or_index; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + stack_pointer[-1] = null_or_index; + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + next = item; + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ + + case _ITER_CHECK_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _ITER_CHECK_LIST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _ITER_CHECK_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _ITER_CHECK_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_EXHAUSTED_LIST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_LIST_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_LIST_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_LIST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + #ifndef Py_GIL_DISABLED + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ + + case _ITER_NEXT_LIST_TIER_TWO_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); + #ifdef Py_GIL_DISABLED + assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) || + _PyObject_GC_IS_SHARED(list_o)); + STAT_INC(FOR_ITER, hit); + stack_pointer[0] = iter; + stack_pointer[1] = null_or_index; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result <= 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #else + assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); + stack_pointer += 2; + #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _MATCH_CLASS: { - _PyStackRef names; - _PyStackRef type; - _PyStackRef subject; - _PyStackRef attrs; - oparg = CURRENT_OPARG(); - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - _PyStackRef tmp = names; - names = PyStackRef_NULL; - stack_pointer[-1] = names; - PyStackRef_CLOSE(tmp); - tmp = type; - type = PyStackRef_NULL; - stack_pointer[-2] = type; - PyStackRef_CLOSE(tmp); - tmp = subject; - subject = PyStackRef_NULL; - stack_pointer[-3] = subject; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) { - JUMP_TO_ERROR(); - } - attrs = PyStackRef_None; + case _ITER_CHECK_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - stack_pointer[0] = attrs; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _MATCH_MAPPING: { - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + case _ITER_CHECK_TUPLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _MATCH_SEQUENCE: { - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + case _ITER_CHECK_TUPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _MATCH_KEYS: { - _PyStackRef keys; - _PyStackRef subject; - _PyStackRef values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) { - JUMP_TO_ERROR(); + case _ITER_CHECK_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); } - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GET_ITER: { - _PyStackRef iterable; - _PyStackRef iter; - _PyStackRef index_or_null; - iterable = stack_pointer[-1]; - #ifdef Py_STATS - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_GatherStats_GetIter(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ - PyTypeObject *tp = PyStackRef_TYPE(iterable); - if (tp == &PyTuple_Type || tp == &PyList_Type) { - iter = iterable; - index_or_null = PyStackRef_TagInt(0); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_ERROR(); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - index_or_null = PyStackRef_NULL; - stack_pointer += 1; + case _GUARD_NOT_EXHAUSTED_TUPLE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - stack_pointer[-1] = iter; - stack_pointer[0] = index_or_null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GET_YIELD_FROM_ITER: { - _PyStackRef iterable; + case _GUARD_NOT_EXHAUSTED_TUPLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; _PyStackRef iter; - iterable = stack_pointer[-1]; - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_ERROR(); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_ERROR(); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = iterable; - iterable = iter; - stack_pointer[-1] = iterable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOT_EXHAUSTED_TUPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } - stack_pointer[-1] = iter; + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ + case _GUARD_NOT_EXHAUSTED_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null_or_index = _stack_item_2; + iter = _stack_item_1; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = null_or_index; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } - case _FOR_ITER_TIER_TWO: { + case _ITER_NEXT_TUPLE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyStackRef_IsValid(item)) { - if (PyStackRef_IsError(item)) { - JUMP_TO_ERROR(); - } - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - next = item; - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null_or_index = _stack_item_1; + iter = _stack_item_0; + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 because it is instrumented */ + case _ITER_CHECK_RANGE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } - case _ITER_CHECK_LIST: { - _PyStackRef null_or_index; + case _ITER_CHECK_RANGE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef iter; - null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyList_Type) { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } #endif + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 because it is replaced */ - - case _GUARD_NOT_EXHAUSTED_LIST: { - _PyStackRef null_or_index; + case _ITER_CHECK_RANGE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef iter; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - #ifndef Py_GIL_DISABLED - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(list_o) == &PyList_Type); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + _PyStackRef _stack_item_0 = _tos_cache0; + iter = stack_pointer[-1]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ - - case _ITER_NEXT_LIST_TIER_TWO: { - _PyStackRef null_or_index; + case _ITER_CHECK_RANGE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef iter; - _PyStackRef next; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(PyList_CheckExact(list_o)); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + iter = _stack_item_1; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } #ifdef Py_GIL_DISABLED - assert(_Py_IsOwnedByCurrentThread((PyObject *)list_o) || - _PyObject_GC_IS_SHARED(list_o)); - STAT_INC(FOR_ITER, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result <= 0) { + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - #else - assert(PyStackRef_UntagInt(null_or_index) < PyList_GET_SIZE(list_o)); - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _ITER_CHECK_TUPLE: { - _PyStackRef null_or_index; + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_EXHAUSTED_RANGE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef iter; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTuple_Type) { + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + if (r->len <= 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - assert(PyStackRef_IsTaggedInt(null_or_index)); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 because it is replaced */ - - case _GUARD_NOT_EXHAUSTED_TUPLE: { - _PyStackRef null_or_index; + case _GUARD_NOT_EXHAUSTED_RANGE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef iter; - null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + if (r->len <= 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _ITER_NEXT_TUPLE: { - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef next; - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - uintptr_t i = PyStackRef_UntagInt(null_or_index); - assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - - case _ITER_CHECK_RANGE: { + case _GUARD_NOT_EXHAUSTED_RANGE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef iter; - iter = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + assert(Py_TYPE(r) == &PyRangeIter_Type); + if (r->len <= 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 because it is replaced */ - - case _GUARD_NOT_EXHAUSTED_RANGE: { + case _GUARD_NOT_EXHAUSTED_RANGE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef iter; - iter = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + iter = _stack_item_1; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); if (r->len <= 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _ITER_NEXT_RANGE: { + case _ITER_NEXT_RANGE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef iter; _PyStackRef next; - iter = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED @@ -4573,34 +9361,48 @@ r->len--; PyObject *res = PyLong_FromLong(value); if (res == NULL) { + stack_pointer[0] = iter; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } next = PyStackRef_FromPyObjectSteal(res); - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = next; + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _FOR_ITER_GEN_FRAME: { + case _FOR_ITER_GEN_FRAME_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef iter; _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - iter = stack_pointer[-2]; + iter = _stack_item_0; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } #endif if (gen->gi_frame_state >= FRAME_EXECUTING) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } STAT_INC(FOR_ITER, hit); @@ -4612,25 +9414,34 @@ pushed_frame->previous = frame; frame->return_offset = (uint16_t)( 2 + oparg); gen_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[0] = gen_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = gen_frame; + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _INSERT_NULL: { + case _INSERT_NULL_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef self; _PyStackRef *method_and_self; - self = stack_pointer[-1]; - method_and_self = &stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + self = _stack_item_0; + method_and_self = &stack_pointer[0]; method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; - stack_pointer += 1; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_SPECIAL: { + case _LOAD_SPECIAL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *method_and_self; oparg = CURRENT_OPARG(); method_and_self = &stack_pointer[-2]; @@ -4652,21 +9463,32 @@ _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); stack_pointer = _PyFrame_GetStackPointer(frame); } + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _WITH_EXCEPT_START: { + case _WITH_EXCEPT_START_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef val; _PyStackRef lasti; _PyStackRef exit_self; _PyStackRef exit_func; _PyStackRef res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_self = stack_pointer[-4]; - exit_func = stack_pointer[-5]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + val = _stack_item_2; + lasti = _stack_item_0; + exit_self = stack_pointer[-1]; + exit_func = stack_pointer[-2]; PyObject *exc, *tb; PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); @@ -4680,22 +9502,59 @@ (void)lasti; PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); + stack_pointer[0] = lasti; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = val; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); Py_XDECREF(original_tb); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; + _tos_cache2 = res; + _tos_cache1 = val; + _tos_cache0 = _stack_item_1; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _PUSH_EXC_INFO_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + _PyStackRef _stack_item_0 = _tos_cache0; + exc = _stack_item_0; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + _tos_cache1 = new_exc; + _tos_cache0 = prev_exc; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _PUSH_EXC_INFO: { + case _PUSH_EXC_INFO_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; @@ -4710,14 +9569,63 @@ assert(PyStackRef_ExceptionInstanceCheck(exc)); exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); new_exc = exc; + _tos_cache0 = new_exc; + SET_CURRENT_CACHED_VALUES(1); stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _PUSH_EXC_INFO_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + exc = _stack_item_1; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + _tos_cache2 = new_exc; + _tos_cache1 = prev_exc; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -4725,12 +9633,84 @@ PyDictValues *ivs = _PyObject_InlineValues(owner_o); if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_KEYS_VERSION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_KEYS_VERSION: { + case _GUARD_KEYS_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); @@ -4739,12 +9719,88 @@ PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_KEYS_VERSION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_KEYS_VERSION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_METHOD_WITH_VALUES: { + case _LOAD_ATTR_METHOD_WITH_VALUES_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -4757,19 +9813,95 @@ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; + _tos_cache0 = self; + SET_CURRENT_CACHED_VALUES(1); stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache2 = self; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + oparg = CURRENT_OPARG(); + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache0 = self; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[-1] = attr; + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_METHOD_NO_DICT: { + case _LOAD_ATTR_METHOD_NO_DICT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + owner = _stack_item_1; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); @@ -4778,57 +9910,86 @@ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = self; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + owner = _stack_item_0; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; _PyStackRef attr; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + owner = _stack_item_0; PyObject *descr = (PyObject *)CURRENT_OPERAND0(); assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = attr; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + owner = _stack_item_0; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = owner; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_ATTR_METHOD_LAZY_DICT: { + case _CHECK_ATTR_METHOD_LAZY_DICT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); @@ -4836,12 +9997,86 @@ PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + owner = _stack_item_2; + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = owner; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + owner = _stack_item_0; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache1 = self; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_ATTR_METHOD_LAZY_DICT: { + case _LOAD_ATTR_METHOD_LAZY_DICT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -4854,14 +10089,41 @@ assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; + _tos_cache0 = self; + SET_CURRENT_CACHED_VALUES(1); stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + owner = _stack_item_1; + PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + _tos_cache2 = self; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _MAYBE_EXPAND_METHOD: { + case _MAYBE_EXPAND_METHOD_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -4880,8 +10142,13 @@ PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } @@ -4889,7 +10156,9 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _PY_FRAME_GENERAL: { + case _PY_FRAME_GENERAL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -4916,16 +10185,21 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } new_frame = PyStackRef_Wrap(temp); - stack_pointer[0] = new_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_FUNCTION_VERSION: { + case _CHECK_FUNCTION_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; @@ -4933,29 +10207,103 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_FUNCTION_VERSION_INLINE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_FUNCTION_VERSION_INLINE: { + case _CHECK_FUNCTION_VERSION_INLINE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_METHOD_VERSION: { + case _CHECK_METHOD_VERSION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -4965,25 +10313,33 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (((PyFunctionObject *)func)->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _EXPAND_METHOD: { + case _EXPAND_METHOD_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -5001,26 +10357,39 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_IS_NOT_PY_CALLABLE: { + case _CHECK_IS_NOT_PY_CALLABLE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_NON_PY_GENERAL: { + case _CALL_NON_PY_GENERAL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5059,6 +10428,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5088,16 +10458,21 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -5105,16 +10480,22 @@ callable = stack_pointer[-2 - oparg]; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -5131,18 +10512,81 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_PEP_523_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + if (tstate->interp->eval_frame) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_PEP_523_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + if (tstate->interp->eval_frame) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_PEP_523_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (tstate->interp->eval_frame) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_PEP_523: { + case _CHECK_PEP_523_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; if (tstate->interp->eval_frame) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_FUNCTION_EXACT_ARGS: { + case _CHECK_FUNCTION_EXACT_ARGS_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -5154,12 +10598,17 @@ PyCodeObject *code = (PyCodeObject *)func->func_code; if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_STACK_SPACE: { + case _CHECK_STACK_SPACE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; @@ -5168,20 +10617,81 @@ PyCodeObject *code = (PyCodeObject *)func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_RECURSION_REMAINING_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_RECURSION_REMAINING_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_RECURSION_REMAINING_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_RECURSION_REMAINING: { + case _CHECK_RECURSION_REMAINING_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _INIT_CALL_PY_EXACT_ARGS_0: { + case _INIT_CALL_PY_EXACT_ARGS_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5200,13 +10710,19 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _INIT_CALL_PY_EXACT_ARGS_1: { + case _INIT_CALL_PY_EXACT_ARGS_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5225,13 +10741,19 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _INIT_CALL_PY_EXACT_ARGS_2: { + case _INIT_CALL_PY_EXACT_ARGS_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5250,13 +10772,19 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _INIT_CALL_PY_EXACT_ARGS_3: { + case _INIT_CALL_PY_EXACT_ARGS_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5275,13 +10803,19 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _INIT_CALL_PY_EXACT_ARGS_4: { + case _INIT_CALL_PY_EXACT_ARGS_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5300,13 +10834,19 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _INIT_CALL_PY_EXACT_ARGS: { + case _INIT_CALL_PY_EXACT_ARGS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5324,19 +10864,24 @@ first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _PUSH_FRAME: { + case _PUSH_FRAME_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef new_frame; - new_frame = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + new_frame = _stack_item_0; assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -5345,98 +10890,285 @@ LOAD_SP(); LOAD_IP(0); LLTRACE_RESUME_FRAME(); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_NULL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null = _stack_item_0; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_NOS_NULL: { + case _GUARD_NOS_NULL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef null; null = stack_pointer[-2]; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_NULL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + null = stack_pointer[-1]; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_NULL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef null; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null = _stack_item_1; + if (!PyStackRef_IsNull(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = null; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_NOT_NULL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_NOS_NOT_NULL: { + case _GUARD_NOS_NOT_NULL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (o == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_NOT_NULL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_NOS_NOT_NULL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_THIRD_NULL: { + case _GUARD_THIRD_NULL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef null; - null = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null = _stack_item_0; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_CALLABLE_TYPE_1: { + case _GUARD_CALLABLE_TYPE_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyType_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_TYPE_1: { + case _CALL_TYPE_1_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef arg; _PyStackRef null; _PyStackRef callable; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; oparg = CURRENT_OPARG(); - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; + arg = _stack_item_2; + null = _stack_item_1; + callable = _stack_item_0; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); (void)callable; (void)null; STAT_INC(CALL, hit); res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); - stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[0] = res; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_CALLABLE_STR_1: { + case _GUARD_CALLABLE_STR_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyUnicode_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_STR_1: { + case _CALL_STR_1_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef arg; _PyStackRef null; _PyStackRef callable; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; oparg = CURRENT_OPARG(); - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; + arg = _stack_item_2; + null = _stack_item_1; + callable = _stack_item_0; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); STAT_INC(CALL, hit); + stack_pointer[0] = callable; + stack_pointer[1] = null; + stack_pointer[2] = arg; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Str(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5448,38 +11180,62 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_CALLABLE_TUPLE_1: { + case _GUARD_CALLABLE_TUPLE_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyTuple_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_TUPLE_1: { + case _CALL_TUPLE_1_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef arg; _PyStackRef null; _PyStackRef callable; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; oparg = CURRENT_OPARG(); - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; + arg = _stack_item_2; + null = _stack_item_1; + callable = _stack_item_0; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); STAT_INC(CALL, hit); + stack_pointer[0] = callable; + stack_pointer[1] = null; + stack_pointer[2] = arg; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5491,16 +11247,21 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_AND_ALLOCATE_OBJECT: { + case _CHECK_AND_ALLOCATE_OBJECT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -5510,15 +11271,18 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!PyType_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyTypeObject *tp = (PyTypeObject *)callable_o; if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } assert(tp->tp_new == PyBaseObject_Type.tp_new); @@ -5529,6 +11293,7 @@ PyCodeObject *code = (PyCodeObject *)init_func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -5536,6 +11301,7 @@ PyObject *self_o = PyType_GenericAlloc(tp, 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (self_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } self_or_null = PyStackRef_FromPyObjectSteal(self_o); @@ -5546,10 +11312,17 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CREATE_INIT_FRAME: { + case _CREATE_INIT_FRAME_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self; _PyStackRef init; @@ -5575,34 +11348,49 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; tstate->py_recursion_remaining--; init_frame = PyStackRef_Wrap(temp); - stack_pointer[0] = init_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = init_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _EXIT_INIT_CHECK: { + case _EXIT_INIT_CHECK_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef should_be_none; - should_be_none = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + should_be_none = _stack_item_0; if (!PyStackRef_IsNone(should_be_none)) { + stack_pointer[0] = should_be_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_BUILTIN_CLASS: { + case _CALL_BUILTIN_CLASS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5614,6 +11402,7 @@ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyType_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyTypeObject *tp = (PyTypeObject *)callable_o; @@ -5625,6 +11414,7 @@ } if (tp->tp_vectorcall == NULL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -5648,6 +11438,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5673,16 +11464,21 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_BUILTIN_O: { + case _CALL_BUILTIN_O_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5699,18 +11495,22 @@ } if (total_args != 1) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (_Py_ReachedRecursionLimit(tstate)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -5730,16 +11530,21 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_BUILTIN_FAST: { + case _CALL_BUILTIN_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5757,10 +11562,12 @@ } if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -5785,6 +11592,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5814,16 +11622,21 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { + case _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -5841,10 +11654,12 @@ } if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -5872,6 +11687,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5898,47 +11714,73 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_CALLABLE_LEN: { + case _GUARD_CALLABLE_LEN_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.len) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_LEN: { + case _CALL_LEN_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef arg; _PyStackRef null; _PyStackRef callable; _PyStackRef res; - arg = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + arg = _stack_item_2; + null = _stack_item_1; + callable = _stack_item_0; (void)null; STAT_INC(CALL, hit); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + stack_pointer[0] = callable; + stack_pointer[1] = null; + stack_pointer[2] = arg; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); Py_ssize_t len_i = PyObject_Length(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } stack_pointer += -1; @@ -5952,41 +11794,65 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_CALLABLE_ISINSTANCE: { + case _GUARD_CALLABLE_ISINSTANCE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; - callable = stack_pointer[-4]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = stack_pointer[-1]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.isinstance) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_ISINSTANCE: { + case _CALL_ISINSTANCE_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef cls; _PyStackRef instance; _PyStackRef null; _PyStackRef callable; _PyStackRef res; - cls = stack_pointer[-1]; - instance = stack_pointer[-2]; - null = stack_pointer[-3]; - callable = stack_pointer[-4]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + cls = _stack_item_2; + instance = _stack_item_1; + null = _stack_item_0; + callable = stack_pointer[-1]; STAT_INC(CALL, hit); PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); + stack_pointer[0] = null; + stack_pointer[1] = instance; + stack_pointer[2] = cls; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int retval = PyObject_IsInstance(inst_o, cls_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } (void)null; @@ -6007,46 +11873,67 @@ stack_pointer = _PyFrame_GetStackPointer(frame); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_CALLABLE_LIST_APPEND: { + case _GUARD_CALLABLE_LIST_APPEND_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callable = _stack_item_0; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.list_append) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = callable; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_LIST_APPEND: { + case _CALL_LIST_APPEND_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef arg; _PyStackRef self; _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; oparg = CURRENT_OPARG(); - arg = stack_pointer[-1]; - self = stack_pointer[-2]; - callable = stack_pointer[-3]; + arg = _stack_item_2; + self = _stack_item_1; + callable = _stack_item_0; assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!PyList_CheckExact(self_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); - stack_pointer += -2; + stack_pointer[0] = callable; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self); @@ -6057,6 +11944,7 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } #if TIER_ONE @@ -6064,10 +11952,17 @@ assert(next_instr->op.code == POP_TOP); SKIP_OVER(1); #endif + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_METHOD_DESCRIPTOR_O: { + case _CALL_METHOD_DESCRIPTOR_O_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -6086,19 +11981,23 @@ PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (total_args != 2) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_O) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (_Py_ReachedRecursionLimit(tstate)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } _PyStackRef arg_stackref = arguments[1]; @@ -6106,6 +12005,7 @@ if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -6136,16 +12036,21 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -6163,16 +12068,19 @@ } if (total_args == 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyTypeObject *d_type = method->d_common.d_type; @@ -6180,6 +12088,7 @@ assert(self != NULL); if (!Py_IS_TYPE(self, d_type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -6204,6 +12113,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6232,16 +12142,21 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_METHOD_DESCRIPTOR_NOARGS: { + case _CALL_METHOD_DESCRIPTOR_NOARGS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -6259,11 +12174,13 @@ } if (total_args != 1) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDef *meth = method->d_method; @@ -6271,14 +12188,17 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (meth->ml_flags != METH_NOARGS) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } if (_Py_ReachedRecursionLimit(tstate)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -6297,16 +12217,21 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_METHOD_DESCRIPTOR_FAST: { + case _CALL_METHOD_DESCRIPTOR_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -6324,22 +12249,26 @@ } if (total_args == 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_FASTCALL) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); @@ -6364,6 +12293,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6391,23 +12321,29 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _MAYBE_EXPAND_METHOD_KW: { + case _MAYBE_EXPAND_METHOD_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -6415,30 +12351,42 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + _tos_cache0 = _stack_item_0; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } /* _DO_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _PY_FRAME_KW: { + case _PY_FRAME_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef kwnames; _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; _PyStackRef new_frame; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + kwnames = _stack_item_0; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; @@ -6451,6 +12399,9 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[0] = kwnames; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, @@ -6465,67 +12416,91 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } new_frame = PyStackRef_Wrap(temp); - stack_pointer[0] = new_frame; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = new_frame; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_FUNCTION_VERSION_KW: { + case _CHECK_FUNCTION_VERSION_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_METHOD_VERSION_KW: { + case _CHECK_METHOD_VERSION_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef null; _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (((PyFunctionObject *)func)->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _EXPAND_METHOD_KW: { + case _EXPAND_METHOD_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef self_or_null; _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; assert(PyStackRef_IsNull(self_or_null)); _PyStackRef callable_s = callable; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -6533,41 +12508,62 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable_s); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = _stack_item_0; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_IS_NOT_PY_CALLABLE_KW: { + case _CHECK_IS_NOT_PY_CALLABLE_KW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callable; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CALL_KW_NON_PY: { + case _CALL_KW_NON_PY_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef kwnames; _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + kwnames = _stack_item_0; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif @@ -6583,7 +12579,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = kwnames; kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; + stack_pointer[0] = kwnames; PyStackRef_CLOSE(tmp); for (int _i = oparg; --_i >= 0;) { tmp = args[_i]; @@ -6592,19 +12588,23 @@ } tmp = self_or_null; self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-1 - oparg] = self_or_null; PyStackRef_XCLOSE(tmp); tmp = callable; callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = callable; PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + stack_pointer[0] = kwnames; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( callable_o, args_o, @@ -6637,32 +12637,47 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _MAKE_CALLARGS_A_TUPLE: { + case _MAKE_CALLARGS_A_TUPLE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef callargs; _PyStackRef func; - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + callargs = _stack_item_1; + func = stack_pointer[-1]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = callargs; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyStackRef temp = callargs; @@ -6671,18 +12686,29 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(temp); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; } - stack_pointer[-2] = callargs; + _tos_cache2 = _stack_item_2; + _tos_cache1 = callargs; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } /* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _MAKE_FUNCTION: { + case _MAKE_FUNCTION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef codeobj_st; _PyStackRef func; - codeobj_st = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + codeobj_st = _stack_item_0; PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + stack_pointer[0] = codeobj_st; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -6693,24 +12719,31 @@ PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (func_obj == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[0] = func; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = func; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _SET_FUNCTION_ATTRIBUTE: { + case _SET_FUNCTION_ATTRIBUTE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef func_in; _PyStackRef attr_st; _PyStackRef func_out; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; + func_in = _stack_item_1; + attr_st = _stack_item_0; PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); func_out = func_in; @@ -6720,13 +12753,17 @@ PyObject **ptr = (PyObject **)(((char *)func) + offset); assert(*ptr == NULL); *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = func_out; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _RETURN_GENERATOR: { + case _RETURN_GENERATOR_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -6734,6 +12771,7 @@ PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } assert(STACK_LEVEL() == 0); @@ -6752,13 +12790,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BUILD_SLICE: { + case _BUILD_SLICE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef *args; _PyStackRef slice; oparg = CURRENT_OPARG(); @@ -6778,23 +12820,32 @@ stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); if (slice_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } slice = PyStackRef_FromPyObjectStealMortal(slice_o); - stack_pointer[0] = slice; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = slice; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CONVERT_VALUE: { + case _CONVERT_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; _PyStackRef result; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - value = stack_pointer[-1]; + value = _stack_item_0; conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6804,21 +12855,30 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = result; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _FORMAT_SIMPLE: { + case _FORMAT_SIMPLE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; _PyStackRef res; - value = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6828,26 +12888,36 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); } else { res = value; - stack_pointer += -1; } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _FORMAT_WITH_SPEC: { + case _FORMAT_WITH_SPEC_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef fmt_spec; _PyStackRef value; _PyStackRef res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + fmt_spec = _stack_item_1; + value = _stack_item_0; + stack_pointer[0] = value; + stack_pointer[1] = fmt_spec; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); _PyStackRef tmp = fmt_spec; @@ -6862,74 +12932,138 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _COPY_1_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + bottom = _stack_item_0; + top = PyStackRef_DUP(bottom); + _tos_cache1 = top; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _COPY_1: { + case _COPY_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1]; top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = top; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _COPY_2: { + case _COPY_1_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef bottom; _PyStackRef top; - bottom = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + bottom = _stack_item_1; top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = top; + _tos_cache1 = bottom; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _COPY_2_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef bottom; + _PyStackRef top; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + bottom = _stack_item_0; + top = PyStackRef_DUP(bottom); + _tos_cache2 = top; + _tos_cache1 = _stack_item_1; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _COPY_3: { + case _COPY_3_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef bottom; _PyStackRef top; - bottom = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + bottom = _stack_item_0; top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; + _tos_cache2 = top; + _tos_cache1 = _stack_item_2; + _tos_cache0 = _stack_item_1; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer[0] = bottom; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _COPY: { + case _COPY_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef bottom; _PyStackRef top; oparg = CURRENT_OPARG(); bottom = stack_pointer[-1 - (oparg-1)]; top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = top; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _BINARY_OP: { + case _BINARY_OP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef rhs; _PyStackRef lhs; _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; + rhs = _stack_item_1; + lhs = _stack_item_0; PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); + stack_pointer[0] = lhs; + stack_pointer[1] = rhs; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -6945,10 +13079,38 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SWAP_2_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + top = _stack_item_1; + bottom = _stack_item_0; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache1 = top; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _SWAP_2: { + case _SWAP_2_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef top; _PyStackRef bottom; top = stack_pointer[-1]; @@ -6956,35 +13118,89 @@ _PyStackRef temp = bottom; bottom = top; top = temp; + SET_CURRENT_CACHED_VALUES(0); stack_pointer[-2] = bottom; stack_pointer[-1] = top; + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _SWAP_3: { + case _SWAP_2_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef top; _PyStackRef bottom; - top = stack_pointer[-1]; - bottom = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + top = _stack_item_0; + bottom = stack_pointer[-1]; _PyStackRef temp = bottom; bottom = top; top = temp; - stack_pointer[-3] = bottom; - stack_pointer[-1] = top; + _tos_cache0 = top; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[-1] = bottom; + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SWAP_2_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + top = _stack_item_2; + bottom = _stack_item_1; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache2 = top; + _tos_cache1 = bottom; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SWAP_3_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef top; + _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + top = _stack_item_2; + bottom = _stack_item_0; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + _tos_cache2 = top; + _tos_cache1 = _stack_item_1; + _tos_cache0 = bottom; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _SWAP: { + case _SWAP_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef top; _PyStackRef bottom; + _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; + top = _stack_item_0; + bottom = stack_pointer[-1 - (oparg-2)]; _PyStackRef temp = bottom; bottom = top; top = temp; - stack_pointer[-2 - (oparg-2)] = bottom; - stack_pointer[-1] = top; + _tos_cache0 = top; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[-1 - (oparg-2)] = bottom; + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } @@ -7006,94 +13222,432 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 because it is instrumented */ - case _GUARD_IS_TRUE_POP: { + case _GUARD_IS_TRUE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef flag; - flag = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + flag = _stack_item_0; + int is_true = PyStackRef_IsTrue(flag); + if (!is_true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_IS_TRUE_POP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + flag = _stack_item_1; + int is_true = PyStackRef_IsTrue(flag); + if (!is_true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_IS_TRUE_POP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + flag = _stack_item_2; int is_true = PyStackRef_IsTrue(flag); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (!is_true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_IS_FALSE_POP: { + case _GUARD_IS_FALSE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef flag; - flag = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + flag = _stack_item_0; + int is_false = PyStackRef_IsFalse(flag); + if (!is_false) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_IS_FALSE_POP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + flag = _stack_item_1; + int is_false = PyStackRef_IsFalse(flag); + if (!is_false) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_IS_FALSE_POP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef flag; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + flag = _stack_item_2; int is_false = PyStackRef_IsFalse(flag); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); if (!is_false) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_IS_NONE_POP: { + case _GUARD_IS_NONE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef val; - val = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + val = _stack_item_0; int is_none = PyStackRef_IsNone(val); if (!is_none) { - stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_IS_NONE_POP_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + val = _stack_item_1; + int is_none = PyStackRef_IsNone(val); + if (!is_none) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(val); stack_pointer = _PyFrame_GetStackPointer(frame); if (1) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _GUARD_IS_NOT_NONE_POP: { + case _GUARD_IS_NONE_POP_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef val; - val = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + val = _stack_item_2; + int is_none = PyStackRef_IsNone(val); + if (!is_none) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(val); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _GUARD_IS_NOT_NONE_POP_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + val = _stack_item_0; int is_none = PyStackRef_IsNone(val); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(val); stack_pointer = _PyFrame_GetStackPointer(frame); if (is_none) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _JUMP_TO_TOP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + JUMP_TO_JUMP_TARGET(); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SET_IP_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SET_IP_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SET_IP_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SET_IP_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_STACK_SPACE_OPERAND_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _JUMP_TO_TOP: { - JUMP_TO_JUMP_TARGET(); + case _CHECK_STACK_SPACE_OPERAND_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _SET_IP: { - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); - frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; + case _CHECK_STACK_SPACE_OPERAND_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + assert(framesize <= INT_MAX); + if (!_PyThreadState_HasStackSpace(tstate, framesize)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_STACK_SPACE_OPERAND: { + case _CHECK_STACK_SPACE_OPERAND_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SAVE_RETURN_OFFSET_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SAVE_RETURN_OFFSET_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SAVE_RETURN_OFFSET_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _SAVE_RETURN_OFFSET: { + case _SAVE_RETURN_OFFSET_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; oparg = CURRENT_OPARG(); #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); @@ -7101,16 +13655,122 @@ #if TIER_TWO frame->return_offset = oparg; #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _EXIT_TRACE: { + case _EXIT_TRACE_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + _PyExitData *exit = (_PyExitData *)exit_p; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %lu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + tstate->jit_exit = exit; + SET_CURRENT_CACHED_VALUES(0); + TIER2_TO_TIER2(exit->executor); + } + + case _EXIT_TRACE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + _PyExitData *exit = (_PyExitData *)exit_p; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %lu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + } + #endif + tstate->jit_exit = exit; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TIER2_TO_TIER2(exit->executor); + } + + case _EXIT_TRACE_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + _PyExitData *exit = (_PyExitData *)exit_p; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("SIDE EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %lu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + } + #endif + tstate->jit_exit = exit; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TIER2_TO_TIER2(exit->executor); + } + + case _EXIT_TRACE_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); @@ -7119,80 +13779,223 @@ (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; } #endif tstate->jit_exit = exit; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); TIER2_TO_TIER2(exit->executor); + } + + case _CHECK_VALIDITY_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_VALIDITY_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_VALIDITY_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_VALIDITY: { + case _CHECK_VALIDITY_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_CONST_INLINE: { + case _LOAD_CONST_INLINE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectNew(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_CONST_INLINE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_CONST_INLINE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_TOP_LOAD_CONST_INLINE: { + case _POP_TOP_LOAD_CONST_INLINE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef pop; _PyStackRef value; - pop = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + pop = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectNew(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_CONST_INLINE_BORROW: { + case _LOAD_CONST_INLINE_BORROW_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_CONST_INLINE_BORROW_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_CONST_INLINE_BORROW_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_CALL: { + case _POP_CALL_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef null; _PyStackRef callable; - null = stack_pointer[-1]; - callable = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null = _stack_item_1; + callable = _stack_item_0; (void)null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_CALL_ONE: { + case _POP_CALL_ONE_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef pop; _PyStackRef null; _PyStackRef callable; - pop = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; - stack_pointer += -1; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + pop = _stack_item_2; + null = _stack_item_1; + callable = _stack_item_0; + stack_pointer[0] = callable; + stack_pointer[1] = null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); @@ -7203,19 +14006,31 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_CALL_TWO: { + case _POP_CALL_TWO_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef pop2; _PyStackRef pop1; _PyStackRef null; _PyStackRef callable; - pop2 = stack_pointer[-1]; - pop1 = stack_pointer[-2]; - null = stack_pointer[-3]; - callable = stack_pointer[-4]; - stack_pointer += -1; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + pop2 = _stack_item_2; + pop1 = _stack_item_1; + null = _stack_item_0; + callable = stack_pointer[-1]; + stack_pointer[0] = null; + stack_pointer[1] = pop1; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); @@ -7231,34 +14046,47 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + case _POP_TOP_LOAD_CONST_INLINE_BORROW_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef pop; _PyStackRef value; - pop = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + pop = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_TWO_LOAD_CONST_INLINE_BORROW: { + case _POP_TWO_LOAD_CONST_INLINE_BORROW_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef pop2; _PyStackRef pop1; _PyStackRef value; - pop2 = stack_pointer[-1]; - pop1 = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + pop2 = _stack_item_1; + pop1 = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - stack_pointer += -1; + stack_pointer[0] = pop1; + stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); @@ -7269,42 +14097,55 @@ PyStackRef_CLOSE(pop1); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_CALL_LOAD_CONST_INLINE_BORROW: { + case _POP_CALL_LOAD_CONST_INLINE_BORROW_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef null; _PyStackRef callable; _PyStackRef value; - null = stack_pointer[-1]; - callable = stack_pointer[-2]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null = _stack_item_1; + callable = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); (void)null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: { + case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef pop; _PyStackRef null; _PyStackRef callable; _PyStackRef value; - pop = stack_pointer[-1]; - null = stack_pointer[-2]; - callable = stack_pointer[-3]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + pop = _stack_item_2; + null = _stack_item_1; + callable = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - stack_pointer += -1; + stack_pointer[0] = callable; + stack_pointer[1] = null; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); @@ -7316,91 +14157,253 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef pop2; + _PyStackRef pop1; + _PyStackRef null; + _PyStackRef callable; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + pop2 = _stack_item_2; + pop1 = _stack_item_1; + null = _stack_item_0; + callable = stack_pointer[-1]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + stack_pointer[0] = null; + stack_pointer[1] = pop1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(pop2); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(pop1); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)null; + stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_CONST_UNDER_INLINE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef old; + _PyStackRef value; + _PyStackRef new; + _PyStackRef _stack_item_0 = _tos_cache0; + old = _stack_item_0; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + new = old; + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache1 = new; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_CONST_UNDER_INLINE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef old; + _PyStackRef value; + _PyStackRef new; + old = stack_pointer[-1]; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + new = old; + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache0 = new; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[-1] = value; + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _LOAD_CONST_UNDER_INLINE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef old; + _PyStackRef value; + _PyStackRef new; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + old = _stack_item_1; + PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + new = old; + value = PyStackRef_FromPyObjectNew(ptr); + _tos_cache2 = new; + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: { - _PyStackRef pop2; - _PyStackRef pop1; - _PyStackRef null; - _PyStackRef callable; + case _LOAD_CONST_UNDER_INLINE_BORROW_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef old; _PyStackRef value; - pop2 = stack_pointer[-1]; - pop1 = stack_pointer[-2]; - null = stack_pointer[-3]; - callable = stack_pointer[-4]; + _PyStackRef new; + _PyStackRef _stack_item_0 = _tos_cache0; + old = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(pop2); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(pop1); - stack_pointer = _PyFrame_GetStackPointer(frame); - (void)null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); + new = old; value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache1 = new; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_CONST_UNDER_INLINE: { + case _LOAD_CONST_UNDER_INLINE_BORROW_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef old; _PyStackRef value; _PyStackRef new; old = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); new = old; - value = PyStackRef_FromPyObjectNew(ptr); + value = PyStackRef_FromPyObjectBorrow(ptr); + _tos_cache0 = new; + SET_CURRENT_CACHED_VALUES(1); stack_pointer[-1] = value; - stack_pointer[0] = new; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _LOAD_CONST_UNDER_INLINE_BORROW: { + case _LOAD_CONST_UNDER_INLINE_BORROW_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef old; _PyStackRef value; _PyStackRef new; - old = stack_pointer[-1]; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + old = _stack_item_1; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); new = old; value = PyStackRef_FromPyObjectBorrow(ptr); - stack_pointer[-1] = value; - stack_pointer[0] = new; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + _tos_cache2 = new; + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_FUNCTION_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_FUNCTION_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _CHECK_FUNCTION_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _CHECK_FUNCTION: { + case _CHECK_FUNCTION_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _START_EXECUTOR: { + case _START_EXECUTOR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + oparg = CURRENT_OPARG(); PyObject *executor = (PyObject *)CURRENT_OPERAND0(); #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); + #ifdef Py_DEBUG + assert(STACK_LEVEL() + current_cached_values == oparg); + #endif #endif assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); tstate->current_executor = (PyObject *)executor; @@ -7412,52 +14415,330 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (true) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MAKE_WARM_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + current_executor->vm_data.warm = true; + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _MAKE_WARM_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + current_executor->vm_data.warm = true; + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _MAKE_WARM: { + case _MAKE_WARM_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; current_executor->vm_data.warm = true; if (--tstate->interp->trace_run_counter == 0) { _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _FATAL_ERROR: { + case _MAKE_WARM_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + current_executor->vm_data.warm = true; + if (--tstate->interp->trace_run_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _FATAL_ERROR_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); assert(0); Py_FatalError("Fatal error uop executed."); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _DEOPT: { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + case _FATAL_ERROR_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + assert(0); + Py_FatalError("Fatal error uop executed."); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _FATAL_ERROR_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + assert(0); + Py_FatalError("Fatal error uop executed."); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _FATAL_ERROR_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + assert(0); + Py_FatalError("Fatal error uop executed."); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _HANDLE_PENDING_AND_DEOPT: { + case _DEOPT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + } + + case _DEOPT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + } + + case _DEOPT_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + } + + case _DEOPT_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + } + + case _HANDLE_PENDING_AND_DEOPT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + } + + case _HANDLE_PENDING_AND_DEOPT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + } + + case _HANDLE_PENDING_AND_DEOPT_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + } + + case _HANDLE_PENDING_AND_DEOPT_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); - break; } - case _ERROR_POP_N: { + case _ERROR_POP_N_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(NULL); + } + + case _TIER2_RESUME_CHECK_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker & _PY_EVAL_EVENTS_MASK) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + SET_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _TIER2_RESUME_CHECK_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker & _PY_EVAL_EVENTS_MASK) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _TIER2_RESUME_CHECK_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + if (eval_breaker & _PY_EVAL_EVENTS_MASK) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _TIER2_RESUME_CHECK: { + case _TIER2_RESUME_CHECK_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -7465,22 +14746,197 @@ uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); if (eval_breaker & _PY_EVAL_EVENTS_MASK) { UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _tos_cache0 = stack_pointer[-1]; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = stack_pointer[-2]; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = stack_pointer[-3]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _tos_cache1 = _stack_item_0; + _tos_cache0 = stack_pointer[-1]; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = stack_pointer[-2]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r20: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _tos_cache0 = _stack_item_1; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = stack_pointer[-1]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = _stack_item_2; + stack_pointer += 3; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r31: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + _tos_cache0 = _stack_item_2; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + break; + } + + case _SPILL_OR_RELOAD_r32: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + _tos_cache1 = _stack_item_2; + _tos_cache0 = _stack_item_1; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer[0] = _stack_item_0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } - case _COLD_EXIT: { + case _COLD_EXIT_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); + SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(target); } - _PyExecutorObject *executor; + _PyExecutorObject *executor = NULL; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; @@ -7497,14 +14953,15 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (optimized <= 0) { exit->temperature = restart_backoff_counter(temperature); + SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(optimized < 0 ? NULL : target); } exit->temperature = initial_temperature_backoff_counter(); } assert(tstate->jit_exit == exit); exit->executor = executor; + SET_CURRENT_CACHED_VALUES(0); TIER2_TO_TIER2(exit->executor); - break; } #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7547eaad125370..c80928739081cf 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5491,13 +5491,10 @@ } DISPATCH_GOTO(); } - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ - DISPATCH(); } TARGET(EXIT_INIT_CHECK) { @@ -7665,8 +7662,6 @@ this_instr[1].counter = initial_jump_backoff_counter(); stack_pointer = _PyFrame_GetStackPointer(frame); assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; TIER1_TO_TIER2(executor); } } @@ -7811,7 +7806,7 @@ _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; - _PyStackRef *attr; + _PyStackRef attr; _PyStackRef *self_or_null; // _SPECIALIZE_LOAD_ATTR { @@ -7834,17 +7829,18 @@ /* Skip 8 cache entries */ // _LOAD_ATTR { - attr = &stack_pointer[-1]; self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { - *attr = PyStackRef_NULL; + _PyCStackRef method; _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); + _PyThreadState_PushCStackRef(tstate, &method); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, &method.ref); stack_pointer = _PyFrame_GetStackPointer(frame); if (is_meth) { - assert(!PyStackRef_IsNull(*attr)); + assert(!PyStackRef_IsNull(method.ref)); self_or_null[0] = owner; + attr = _PyThreadState_PopCStackRefSteal(tstate, &method); } else { stack_pointer += -1; @@ -7852,10 +7848,11 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*attr)) { + self_or_null[0] = PyStackRef_NULL; + attr = _PyThreadState_PopCStackRefSteal(tstate, &method); + if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(error); } - self_or_null[0] = PyStackRef_NULL; stack_pointer += 1; } } @@ -7871,10 +7868,11 @@ if (attr_o == NULL) { JUMP_TO_LABEL(error); } - *attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer += 1; } } + stack_pointer[-1] = attr; stack_pointer += (oparg&1); assert(WITHIN_STACK_BOUNDS()); DISPATCH(); diff --git a/Python/jit.c b/Python/jit.c index 01ec9c1fa6e8a9..1a8910fb175226 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -502,7 +502,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz data_size += group->data_size; combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); } - group = &stencil_groups[_FATAL_ERROR]; + group = &stencil_groups[_FATAL_ERROR_r00]; code_size += group->code_size; data_size += group->data_size; combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); @@ -535,7 +535,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz unsigned char *code = memory; state.trampolines.mem = memory + code_size; unsigned char *data = memory + code_size + state.trampolines.size + code_padding; - assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); + assert(trace[0].opcode == _START_EXECUTOR_r00 || trace[0].opcode == _COLD_EXIT_r00); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; @@ -544,7 +544,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz data += group->data_size; } // Protect against accidental buffer overrun into data: - group = &stencil_groups[_FATAL_ERROR]; + group = &stencil_groups[_FATAL_ERROR_r00]; group->emit(code, data, executor, NULL, &state); code += group->code_size; data += group->data_size; diff --git a/Python/optimizer.c b/Python/optimizer.c index bae5cfa50ead58..418ec63d39b6bc 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -24,6 +24,7 @@ #define NEED_OPCODE_METADATA #include "pycore_uop_metadata.h" // Uop tables + #undef NEED_OPCODE_METADATA #define MAX_EXECUTORS_SIZE 256 @@ -98,9 +99,6 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO instr->op.arg = index; } -static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); - static int uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, int curr_stackentries, @@ -206,8 +204,8 @@ static int executor_clear(PyObject *executor); static void unlink_executor(_PyExecutorObject *executor); -void -_PyExecutor_Free(_PyExecutorObject *self) +static void +free_executor(_PyExecutorObject *self) { #ifdef _Py_JIT _PyJIT_Free(self); @@ -243,7 +241,7 @@ _Py_ClearExecutorDeletionList(PyInterpreterState *interp) } else { *prev_to_next_ptr = exec->vm_data.links.next; - _PyExecutor_Free(exec); + free_executor(exec); } exec = *prev_to_next_ptr; } @@ -278,7 +276,7 @@ uop_dealloc(PyObject *op) { const char * _PyUOpName(int index) { - if (index < 0 || index > MAX_UOP_ID) { + if (index < 0 || index > MAX_UOP_REGS_ID) { return NULL; } return _PyOpcode_uop_name[index]; @@ -337,7 +335,9 @@ uop_item(PyObject *op, Py_ssize_t index) PyErr_SetNone(PyExc_IndexError); return NULL; } - const char *name = _PyUOpName(self->trace[index].opcode); + int opcode = self->trace[index].opcode; + int base_opcode = _PyUop_Uncached[opcode]; + const char *name = _PyUOpName(base_opcode); if (name == NULL) { name = ""; } @@ -550,7 +550,9 @@ translate_bytecode_to_trace( _Py_CODEUNIT *instr, _PyUOpInstruction *trace, int buffer_size, - _PyBloomFilter *dependencies, bool progress_needed) + _PyBloomFilter *dependencies, + bool progress_needed, + int curr_stackentries) { bool first = true; PyCodeObject *code = _PyFrame_GetCode(frame); @@ -585,7 +587,7 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); - ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); + ADD_TO_TRACE(_START_EXECUTOR, curr_stackentries, (uintptr_t)instr, INSTR_IP(instr, code)); ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); uint32_t target = 0; @@ -1000,16 +1002,36 @@ count_exits(_PyUOpInstruction *buffer, int length) { int exit_count = 0; for (int i = 0; i < length; i++) { - int opcode = buffer[i].opcode; - if (opcode == _EXIT_TRACE) { + uint16_t base_opcode = _PyUop_Uncached[buffer[i].opcode]; + if (base_opcode == _EXIT_TRACE) { exit_count++; } } return exit_count; } +/* The number of cached registers at any exit (`EXIT_IF` or `DEOPT_IF`) + * This is the number of cached at entries at start, unless the uop is + * marked as `exit_depth_is_output` in which case it is the number of + * cached entries at the end */ +static int +get_cached_entries_for_side_exit(_PyUOpInstruction *inst) +{ + // Maybe add another generated table for this? + int base_opcode = _PyUop_Uncached[inst->opcode]; + assert(base_opcode != 0); + for (int i = 0; i <= MAX_CACHED_REGISTER; i++) { + const _PyUopTOSentry *entry = &_PyUop_Caching[base_opcode].entries[i]; + if (entry->opcode == inst->opcode) { + return entry->exit; + } + } + Py_UNREACHABLE(); +} + static void make_exit(_PyUOpInstruction *inst, int opcode, int target) { + assert(opcode > MAX_UOP_ID && opcode <= MAX_UOP_REGS_ID); inst->opcode = opcode; inst->oparg = 0; inst->operand0 = 0; @@ -1046,19 +1068,22 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int next_spare = length; for (int i = 0; i < length; i++) { _PyUOpInstruction *inst = &buffer[i]; - int opcode = inst->opcode; + int base_opcode = _PyUop_Uncached[inst->opcode]; + assert(inst->opcode != _NOP); int32_t target = (int32_t)uop_get_target(inst); - uint16_t exit_flags = _PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG | HAS_PERIODIC_FLAG); + uint16_t exit_flags = _PyUop_Flags[base_opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG | HAS_PERIODIC_FLAG); if (exit_flags) { - uint16_t exit_op = _EXIT_TRACE; + uint16_t base_exit_op = _EXIT_TRACE; if (exit_flags & HAS_DEOPT_FLAG) { - exit_op = _DEOPT; + base_exit_op = _DEOPT; } else if (exit_flags & HAS_PERIODIC_FLAG) { - exit_op = _HANDLE_PENDING_AND_DEOPT; + base_exit_op = _HANDLE_PENDING_AND_DEOPT; } + int exit_depth = get_cached_entries_for_side_exit(inst); + uint16_t exit_op = _PyUop_Caching[base_exit_op].entries[exit_depth].opcode; int32_t jump_target = target; - if (is_for_iter_test[opcode]) { + if (is_for_iter_test[base_opcode]) { /* Target the POP_TOP immediately after the END_FOR, * leaving only the iterator on the stack. */ int extended_arg = inst->oparg > 255; @@ -1075,14 +1100,14 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) buffer[i].jump_target = current_jump; buffer[i].format = UOP_FORMAT_JUMP; } - if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { - int popped = (_PyUop_Flags[opcode] & HAS_ERROR_NO_POP_FLAG) ? - 0 : _PyUop_num_popped(opcode, inst->oparg); + if (_PyUop_Flags[base_opcode] & HAS_ERROR_FLAG) { + int popped = (_PyUop_Flags[base_opcode] & HAS_ERROR_NO_POP_FLAG) ? + 0 : _PyUop_num_popped(base_opcode, inst->oparg); if (target != current_error_target || popped != current_popped) { current_popped = popped; current_error = next_spare; current_error_target = target; - make_exit(&buffer[next_spare], _ERROR_POP_N, 0); + make_exit(&buffer[next_spare], _ERROR_POP_N_r00, 0); buffer[next_spare].operand0 = target; next_spare++; } @@ -1092,8 +1117,8 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) buffer[i].jump_target = 0; } } - if (opcode == _JUMP_TO_TOP) { - assert(buffer[0].opcode == _START_EXECUTOR); + if (base_opcode == _JUMP_TO_TOP) { + assert(_PyUop_Uncached[buffer[0].opcode] == _START_EXECUTOR); buffer[i].format = UOP_FORMAT_JUMP; buffer[i].jump_target = 1; } @@ -1140,25 +1165,30 @@ sanity_check(_PyExecutorObject *executor) } bool ended = false; uint32_t i = 0; - CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT); + CHECK(_PyUop_Uncached[executor->trace[0].opcode] == _START_EXECUTOR || + _PyUop_Uncached[executor->trace[0].opcode] == _COLD_EXIT); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; - CHECK(opcode <= MAX_UOP_ID); - CHECK(_PyOpcode_uop_name[opcode] != NULL); + uint16_t base_opcode = _PyUop_Uncached[opcode]; + CHECK(opcode > MAX_UOP_ID); + CHECK(opcode <= MAX_UOP_REGS_ID); + CHECK(base_opcode <= MAX_UOP_ID); + CHECK(base_opcode != 0); + CHECK(_PyOpcode_uop_name[base_opcode] != NULL); switch(inst->format) { case UOP_FORMAT_TARGET: - CHECK(target_unused(opcode)); + CHECK(target_unused(base_opcode)); break; case UOP_FORMAT_JUMP: CHECK(inst->jump_target < executor->code_size); break; } - if (_PyUop_Flags[opcode] & HAS_ERROR_FLAG) { + if (_PyUop_Flags[base_opcode] & HAS_ERROR_FLAG) { CHECK(inst->format == UOP_FORMAT_JUMP); CHECK(inst->error_target < executor->code_size); } - if (is_terminator(inst)) { + if (base_opcode == _EXIT_TRACE || base_opcode == _JUMP_TO_TOP) { ended = true; i++; break; @@ -1167,12 +1197,12 @@ sanity_check(_PyExecutorObject *executor) CHECK(ended); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; - uint16_t opcode = inst->opcode; + uint16_t base_opcode = _PyUop_Uncached[inst->opcode]; CHECK( - opcode == _DEOPT || - opcode == _HANDLE_PENDING_AND_DEOPT || - opcode == _EXIT_TRACE || - opcode == _ERROR_POP_N); + base_opcode == _DEOPT || + base_opcode == _HANDLE_PENDING_AND_DEOPT || + base_opcode == _EXIT_TRACE || + base_opcode == _ERROR_POP_N); } } @@ -1202,23 +1232,24 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; - assert(buffer[0].opcode == _START_EXECUTOR); + assert(_PyUop_Uncached[buffer[0].opcode] == _START_EXECUTOR); buffer[0].operand0 = (uint64_t)executor; for (int i = length-1; i >= 0; i--) { - int opcode = buffer[i].opcode; + uint16_t base_opcode = _PyUop_Uncached[buffer[i].opcode]; dest--; *dest = buffer[i]; - assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); - if (opcode == _EXIT_TRACE) { + assert(base_opcode != _POP_JUMP_IF_FALSE && base_opcode != _POP_JUMP_IF_TRUE); + if (base_opcode == _EXIT_TRACE) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; + exit->executor = cold; dest->operand0 = (uint64_t)exit; next_exit--; } } assert(next_exit == -1); assert(dest == executor->trace); - assert(dest->opcode == _START_EXECUTOR); + assert(_PyUop_Uncached[dest->opcode] == _START_EXECUTOR); _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -1271,6 +1302,42 @@ int effective_trace_length(_PyUOpInstruction *buffer, int length) } #endif +static int +stack_allocate(_PyUOpInstruction *buffer, int length) +{ + assert(buffer[0].opcode == _START_EXECUTOR); + for (int i = length-1; i >= 0; i--) { + buffer[i*2+1] = buffer[i]; + buffer[i*2].format = UOP_FORMAT_TARGET; + buffer[i*2].oparg = 0; + buffer[i*2].target = 0; + } + int depth = 0; + for (int i = 0; i < length; i++) { + _PyUOpInstruction *spill_or_reload = &buffer[i*2]; + int uop = buffer[i*2+1].opcode; + if (uop == _NOP) { + // leave _NOPs to be cleaned up later + spill_or_reload->opcode = _NOP; + continue; + } + int new_depth = _PyUop_Caching[uop].best[depth]; + if (new_depth == depth) { + spill_or_reload->opcode = _NOP; + } + else { + spill_or_reload->opcode = _PyUop_SpillsAndReloads[depth][new_depth]; + depth = new_depth; + } + uint16_t new_opcode = _PyUop_Caching[uop].entries[depth].opcode; + assert(new_opcode != 0); + assert(spill_or_reload->opcode != 0); + buffer[i*2+1].opcode = new_opcode; + depth = _PyUop_Caching[uop].entries[depth].output; + } + return length*2; +} + static int uop_optimize( _PyInterpreterFrame *frame, @@ -1283,12 +1350,12 @@ uop_optimize( _Py_BloomFilter_Init(&dependencies); _PyUOpInstruction buffer[UOP_MAX_TRACE_LENGTH]; OPT_STAT_INC(attempts); - int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); + int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH/2, &dependencies, progress_needed, curr_stackentries); if (length <= 0) { // Error or nothing translated return length; } - assert(length < UOP_MAX_TRACE_LENGTH); + assert(length < UOP_MAX_TRACE_LENGTH/2); OPT_STAT_INC(traces_created); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); if (env_var == NULL || *env_var == '\0' || *env_var > '0') { @@ -1299,7 +1366,7 @@ uop_optimize( return length; } } - assert(length < UOP_MAX_TRACE_LENGTH); + assert(length < UOP_MAX_TRACE_LENGTH/2); assert(length >= 1); /* Fix up */ for (int pc = 0; pc < length; pc++) { @@ -1315,6 +1382,7 @@ uop_optimize( assert(_PyOpcode_uop_name[buffer[pc].opcode]); } OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); + length = stack_allocate(buffer, length); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); @@ -1484,9 +1552,9 @@ _PyExecutor_GetColdExecutor(void) } _PyExecutorObject *cold = allocate_executor(0, 1); if (cold == NULL) { - Py_FatalError("Cannot allocate core JIT code"); + goto fail; } - ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_EXIT; + ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_EXIT_r00; #ifdef _Py_JIT cold->jit_code = NULL; cold->jit_size = 0; @@ -1494,13 +1562,24 @@ _PyExecutor_GetColdExecutor(void) // from being immediately detected as cold and invalidated. cold->vm_data.warm = true; if (_PyJIT_Compile(cold, cold->trace, 1)) { - Py_DECREF(cold); - Py_FatalError("Cannot allocate core JIT code"); + goto fail; } #endif _Py_SetImmortal((PyObject *)cold); interp->cold_executor = cold; return cold; +fail: + if (cold != NULL) { + free_executor(cold); + } + Py_FatalError("Cannot allocate core JIT code"); +} + +void +_PyExecutor_FreeColdExecutor(_PyExecutorObject *cold) +{ + assert(_PyInterpreterState_GET()->cold_executor != cold); + free_executor(cold); } void @@ -1737,13 +1816,14 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) * https://graphviz.readthedocs.io/en/stable/manual.html#node-ports-compass */ _PyUOpInstruction const *inst = &executor->trace[i]; - const char *opname = _PyOpcode_uop_name[inst->opcode]; + uint16_t base_opcode = _PyUop_Uncached[inst->opcode]; + const char *opname = _PyOpcode_uop_name[base_opcode]; #ifdef Py_STATS fprintf(out, " %s -- %" PRIu64 "\n", i, opname, inst->execution_count); #else fprintf(out, " %s\n", i, opname); #endif - if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { + if (base_opcode == _EXIT_TRACE || base_opcode == _JUMP_TO_TOP) { break; } } @@ -1753,21 +1833,22 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) /* Write all the outgoing edges */ for (uint32_t i = 0; i < executor->code_size; i++) { _PyUOpInstruction const *inst = &executor->trace[i]; - uint16_t flags = _PyUop_Flags[inst->opcode]; + uint16_t base_opcode = _PyUop_Uncached[inst->opcode]; + uint16_t flags = _PyUop_Flags[base_opcode]; _PyExitData *exit = NULL; - if (inst->opcode == _EXIT_TRACE) { + if (base_opcode == _EXIT_TRACE) { exit = (_PyExitData *)inst->operand0; } else if (flags & HAS_EXIT_FLAG) { assert(inst->format == UOP_FORMAT_JUMP); _PyUOpInstruction const *exit_inst = &executor->trace[inst->jump_target]; - assert(exit_inst->opcode == _EXIT_TRACE); + assert(_PyUop_Uncached[exit_inst->opcode] == _EXIT_TRACE); exit = (_PyExitData *)exit_inst->operand0; } if (exit != NULL && exit->executor != NULL) { fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); } - if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { + if (base_opcode == _EXIT_TRACE || base_opcode == _JUMP_TO_TOP) { break; } } @@ -1798,7 +1879,7 @@ _PyDumpExecutors(FILE *out) } void -_PyExecutor_Free(struct _PyExecutorObject *self) +_PyExecutor_FreeColdExecutor(struct _PyExecutorObject *cold) { /* This should never be called */ Py_UNREACHABLE(); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 77759f67532f80..df872f4ddfd433 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -576,9 +576,9 @@ dummy_func(void) { } } - op(_LOAD_ATTR, (owner -- attr[1], self_or_null[oparg&1])) { + op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { (void)owner; - *attr = sym_new_not_null(ctx); + attr = sym_new_not_null(ctx); if (oparg & 1) { self_or_null[0] = sym_new_unknown(ctx); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 2477ede3e68017..eda7eec8e37d01 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1472,16 +1472,16 @@ case _LOAD_ATTR: { JitOptRef owner; - JitOptRef *attr; + JitOptRef attr; JitOptRef *self_or_null; owner = stack_pointer[-1]; - attr = &stack_pointer[-1]; self_or_null = &stack_pointer[0]; (void)owner; - *attr = sym_new_not_null(ctx); + attr = sym_new_not_null(ctx); if (oparg & 1) { self_or_null[0] = sym_new_unknown(ctx); } + stack_pointer[-1] = attr; stack_pointer += (oparg&1); assert(WITHIN_STACK_BOUNDS()); break; @@ -3212,6 +3212,10 @@ break; } + case _SPILL_OR_RELOAD: { + break; + } + case _COLD_EXIT: { break; } diff --git a/Python/pystate.c b/Python/pystate.c index 2465d8667472dc..28f627605604d6 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -821,9 +821,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) struct _PyExecutorObject *cold = interp->cold_executor; if (cold != NULL) { interp->cold_executor = NULL; - assert(cold->vm_data.valid); - assert(cold->vm_data.warm); - _PyExecutor_Free(cold); + _PyExecutor_FreeColdExecutor(cold); } /* We don't clear sysdict and builtins until the end of this function. Because clearing other attributes can execute arbitrary Python code diff --git a/Python/specialize.c b/Python/specialize.c index 38df5741f32520..b7f4cd92d80002 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -14,7 +14,7 @@ #include "pycore_object.h" #include "pycore_opcode_metadata.h" // _PyOpcode_Caches #include "pycore_uop_metadata.h" // _PyOpcode_uop_name -#include "pycore_uop_ids.h" // MAX_UOP_ID +#include "pycore_uop_ids.h" // MAX_UOP_REGS_ID #include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() #include "pycore_runtime.h" // _Py_ID() @@ -33,8 +33,8 @@ GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats }; PyStats *_Py_stats = NULL; -#if PYSTATS_MAX_UOP_ID < MAX_UOP_ID -#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_ID" +#if PYSTATS_MAX_UOP_ID < MAX_UOP_REGS_ID +#error "Not enough space allocated for pystats. Increase PYSTATS_MAX_UOP_ID to at least MAX_UOP_REGS_ID" #endif #define ADD_STAT_TO_DICT(res, field) \ @@ -277,7 +277,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) stats->optimizer_failure_reason_no_memory); fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); - for (int i = 0; i <= MAX_UOP_ID; i++) { + for (int i = 0; i <= MAX_UOP_REGS_ID; i++) { if (stats->opcode[i].execution_count) { fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); } @@ -296,15 +296,15 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) } } - for (int i = 1; i <= MAX_UOP_ID; i++){ - for (int j = 1; j <= MAX_UOP_ID; j++) { + for (int i = 1; i <= MAX_UOP_REGS_ID; i++){ + for (int j = 1; j <= MAX_UOP_REGS_ID; j++) { if (stats->opcode[i].pair_count[j]) { fprintf(out, "uop[%s].pair_count[%s] : %" PRIu64 "\n", _PyOpcode_uop_name[i], _PyOpcode_uop_name[j], stats->opcode[i].pair_count[j]); } } } - for (int i = 0; i < MAX_UOP_ID; i++) { + for (int i = 0; i < MAX_UOP_REGS_ID; i++) { if (stats->error_in_opcode[i]) { fprintf( out, diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 9dd7e5dbfbae7b..a17635b74532cf 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -3,10 +3,12 @@ import lexer import parser import re -from typing import Optional, Callable +from typing import Optional, Callable, Iterator from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, WhileStmt, ForStmt, MacroIfStmt +MAX_CACHED_REGISTER = 3 + @dataclass class EscapingCall: stmt: SimpleStmt @@ -26,12 +28,13 @@ class Properties: eval_breaker: bool needs_this: bool always_exits: bool - stores_sp: bool + sync_sp: bool uses_co_consts: bool uses_co_names: bool uses_locals: bool has_free: bool side_exit: bool + side_exit_at_end: bool pure: bool uses_opcode: bool tier: int | None = None @@ -65,13 +68,14 @@ def from_list(properties: list["Properties"]) -> "Properties": eval_breaker=any(p.eval_breaker for p in properties), needs_this=any(p.needs_this for p in properties), always_exits=any(p.always_exits for p in properties), - stores_sp=any(p.stores_sp for p in properties), + sync_sp=any(p.sync_sp for p in properties), uses_co_consts=any(p.uses_co_consts for p in properties), uses_co_names=any(p.uses_co_names for p in properties), uses_locals=any(p.uses_locals for p in properties), uses_opcode=any(p.uses_opcode for p in properties), has_free=any(p.has_free for p in properties), side_exit=any(p.side_exit for p in properties), + side_exit_at_end=any(p.side_exit_at_end for p in properties), pure=all(p.pure for p in properties), needs_prev=any(p.needs_prev for p in properties), no_save_ip=all(p.no_save_ip for p in properties), @@ -93,13 +97,14 @@ def infallible(self) -> bool: eval_breaker=False, needs_this=False, always_exits=False, - stores_sp=False, + sync_sp=False, uses_co_consts=False, uses_co_names=False, uses_locals=False, uses_opcode=False, has_free=False, side_exit=False, + side_exit_at_end=False, pure=True, no_save_ip=False, ) @@ -692,6 +697,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_Wrap", "PyStackRef_Unwrap", "_PyLong_CheckExactAndCompact", + "_PyThreadState_PopCStackRefSteal", ) @@ -892,7 +898,8 @@ def compute_properties(op: parser.CodeDef) -> Properties: or variable_used(op, "PyCell_SwapTakeRef") ) deopts_if = variable_used(op, "DEOPT_IF") - exits_if = variable_used(op, "EXIT_IF") or variable_used(op, "AT_END_EXIT_IF") + exits_if = variable_used(op, "EXIT_IF") + exit_if_at_end = variable_used(op, "AT_END_EXIT_IF") deopts_periodic = variable_used(op, "HANDLE_PENDING_AND_DEOPT_IF") exits_and_deopts = sum((deopts_if, exits_if, deopts_periodic)) if exits_and_deopts > 1: @@ -917,12 +924,13 @@ def compute_properties(op: parser.CodeDef) -> Properties: deopts=deopts_if, deopts_periodic=deopts_periodic, side_exit=exits_if, + side_exit_at_end=exit_if_at_end, oparg=oparg_used(op), jumps=variable_used(op, "JUMPBY"), eval_breaker="CHECK_PERIODIC" in op.name, needs_this=variable_used(op, "this_instr"), always_exits=always_exits(op), - stores_sp=variable_used(op, "SYNC_SP"), + sync_sp=variable_used(op, "SYNC_SP"), uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), uses_co_names=variable_used(op, "FRAME_CO_NAMES"), uses_locals=variable_used(op, "GETLOCAL") and not has_free, @@ -1280,6 +1288,70 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: instructions, uops, families, pseudos, labels, opmap, first_arg, min_instrumented ) +#Simple heuristic for size to avoid too much stencil duplication +def is_large(uop: Uop) -> bool: + return len(list(uop.body.tokens())) > 80 + +def get_uop_cache_depths(uop: Uop) -> Iterator[tuple[int, int, int]]: + if uop.name == "_SPILL_OR_RELOAD": + for inputs in range(MAX_CACHED_REGISTER+1): + for outputs in range(MAX_CACHED_REGISTER+1): + if inputs != outputs: + yield inputs, outputs, inputs + return + if uop.name in ("_DEOPT", "_HANDLE_PENDING_AND_DEOPT", "_EXIT_TRACE"): + for i in range(MAX_CACHED_REGISTER+1): + yield i, 0, 0 + return + if uop.name in ("_START_EXECUTOR", "_JUMP_TO_TOP", "_COLD_EXIT"): + yield 0, 0, 0 + return + if uop.name == "_ERROR_POP_N": + yield 0, 0, 0 + return + non_decref_escape = False + for call in uop.properties.escaping_calls.values(): + if "DECREF" in call.call.text or "CLOSE" in call.call.text: + continue + non_decref_escape = True + ideal_inputs = 0 + has_array = False + for item in reversed(uop.stack.inputs): + if item.size: + has_array = True + break + ideal_inputs += 1 + ideal_outputs = 0 + for item in reversed(uop.stack.outputs): + if item.size: + has_array = True + break + ideal_outputs += 1 + if ideal_inputs > MAX_CACHED_REGISTER: + ideal_inputs = MAX_CACHED_REGISTER + if ideal_outputs > MAX_CACHED_REGISTER: + ideal_outputs = MAX_CACHED_REGISTER + if non_decref_escape: + yield ideal_inputs, ideal_outputs, 0 + return + at_end = uop.properties.sync_sp or uop.properties.side_exit_at_end + exit_depth = ideal_outputs if at_end else ideal_inputs + yield ideal_inputs, ideal_outputs, exit_depth + if uop.properties.escapes or uop.properties.sync_sp or has_array or is_large(uop): + return + if ideal_inputs == MAX_CACHED_REGISTER or ideal_outputs == MAX_CACHED_REGISTER: + return + inputs, outputs = ideal_inputs, ideal_outputs + if inputs < outputs: + inputs, outputs = 0, outputs-inputs + else: + inputs, outputs = inputs-outputs, 0 + while inputs <= MAX_CACHED_REGISTER and outputs <= MAX_CACHED_REGISTER: + if inputs != ideal_inputs: + yield inputs, outputs, outputs if at_end else inputs + inputs += 1 + outputs += 1 + def analyze_files(filenames: list[str]) -> Analysis: return analyze_forest(parser.parse_files(filenames)) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index cc0edcdf6006fd..bc88458b488613 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -127,6 +127,10 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, + "Py_UNREACHABLE": self.unreachable, + "TIER1_TO_TIER2": self.tier1_to_tier2, + "TIER2_TO_TIER2": self.tier2_to_tier2, + "GOTO_TIER_ONE": self.goto_tier_one } self.out = out self.labels = labels @@ -146,6 +150,19 @@ def dispatch( self.emit(tkn) return False + def unreachable( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.emit(tkn) + emit_to(self.out, tkn_iter, "SEMI") + self.emit(";\n") + return False + def deopt_if( self, tkn: Token, @@ -183,6 +200,16 @@ def periodic_if( ) -> bool: raise NotImplementedError("HANDLE_PENDING_AND_DEOPT_IF not support in tier 1") + def goto_tier_one( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + raise NotImplementedError("GOTO_TIER_ONE not supported in tier 1") + def exit_if_after( self, tkn: Token, @@ -192,8 +219,6 @@ def exit_if_after( inst: Instruction | None, ) -> bool: storage.clear_inputs("in AT_END_EXIT_IF") - storage.flush(self.out) - storage.stack.clear(self.out) return self.exit_if(tkn, tkn_iter, uop, storage, inst) def goto_error(self, offset: int, storage: Storage) -> str: @@ -238,6 +263,24 @@ def error_if( self.out.emit("}\n") return not unconditional + def tier1_to_tier2( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit(tkn) + lparen = next(tkn_iter) + assert lparen.kind == "LPAREN" + self.emit(lparen) + emit_to(self.out, tkn_iter, "RPAREN") + self.out.emit(")") + return False + + tier2_to_tier2 = tier1_to_tier2 + def error_no_pop( self, tkn: Token, @@ -558,10 +601,10 @@ def emit_MacroIfStmt( self.out.emit(stmt.condition) branch = stmt.else_ is not None reachable = True - if branch: - else_storage = storage.copy() + if_storage = storage + else_storage = storage.copy() for s in stmt.body: - r, tkn, storage = self._emit_stmt(s, uop, storage, inst) + r, tkn, if_storage = self._emit_stmt(s, uop, if_storage, inst) if tkn is not None: self.out.emit(tkn) if not r: @@ -576,7 +619,10 @@ def emit_MacroIfStmt( self.out.emit(tkn) if not r: reachable = False - else_storage.merge(storage, self.out) # type: ignore[possibly-undefined] + else_storage.merge(if_storage, self.out) + storage = if_storage + else: + if_storage.merge(else_storage, self.out) storage = else_storage self.out.emit(stmt.endif) return reachable, None, storage @@ -719,7 +765,7 @@ def cflags(p: Properties) -> str: flags.append("HAS_DEOPT_FLAG") if p.deopts_periodic: flags.append("HAS_PERIODIC_FLAG") - if p.side_exit: + if p.side_exit or p.side_exit_at_end: flags.append("HAS_EXIT_FLAG") if not p.infallible: flags.append("HAS_ERROR_FLAG") @@ -731,6 +777,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.no_save_ip: flags.append("HAS_NO_SAVE_IP_FLAG") + if p.sync_sp: + flags.append("HAS_SYNC_SP_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index b649b38123388d..7a938986df4d1b 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -53,6 +53,7 @@ "ESCAPES", "EXIT", "PURE", + "SYNC_SP", "ERROR_NO_POP", "NO_SAVE_IP", "PERIODIC", diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 3a0e7e5d0d5636..e5e0bbb65f482f 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -222,6 +222,13 @@ def __init__(self) -> None: self.logical_sp = PointerOffset.zero() self.variables: list[Local] = [] + + def push_cache(self, cached_items:list[str], out: CWriter) -> None: + for i, name in enumerate(cached_items): + out.start_line() + out.emit(f"_PyStackRef _stack_item_{i} = {name};\n") + self.push(Local.register(f"_stack_item_{i}")) + def drop(self, var: StackItem, check_liveness: bool) -> None: self.logical_sp = self.logical_sp.pop(var) if self.variables: @@ -475,7 +482,7 @@ def clear_dead_inputs(self) -> None: def _push_defined_outputs(self) -> None: defined_output = "" for output in self.outputs: - if output.in_local and not output.memory_offset: + if output.in_local and not output.memory_offset and not output.item.peek: defined_output = output.name if not defined_output: return @@ -499,6 +506,7 @@ def locals_cached(self) -> bool: return False def flush(self, out: CWriter) -> None: + self._print(out) self.clear_dead_inputs() self._push_defined_outputs() self.stack.flush(out) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 1bb5f48658ddfc..0dd87fb7cbd4b4 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -14,6 +14,9 @@ analyze_files, StackItem, analysis_error, + get_uop_cache_depths, + is_large, + MAX_CACHED_REGISTER, ) from generators_common import ( DEFAULT_INPUT, @@ -60,36 +63,19 @@ def declare_variables(uop: Uop, out: CWriter) -> None: class Tier2Emitter(Emitter): - def __init__(self, out: CWriter, labels: dict[str, Label]): + def __init__(self, out: CWriter, labels: dict[str, Label], exit_cache_depth: int): super().__init__(out, labels) self._replacers["oparg"] = self.oparg + self.exit_cache_depth = exit_cache_depth def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. if offset != 0: storage.copy().flush(self.out) - return f"JUMP_TO_ERROR();" - - def deopt_if( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - self.out.emit_at("if ", tkn) - lparen = next(tkn_iter) - self.emit(lparen) - assert lparen.kind == "LPAREN" - first_tkn = tkn_iter.peek() - emit_to(self.out, tkn_iter, "RPAREN") - next(tkn_iter) # Semi colon - self.emit(") {\n") - self.emit("UOP_STAT_INC(uopcode, miss);\n") - self.emit("JUMP_TO_JUMP_TARGET();\n") - self.emit("}\n") - return not always_true(first_tkn) + else: + storage.stack.copy().flush(self.out) + self.emit("SET_CURRENT_CACHED_VALUES(0);\n") + return "JUMP_TO_ERROR();" def exit_if( self, @@ -107,11 +93,12 @@ def exit_if( next(tkn_iter) # Semi colon self.emit(") {\n") self.emit("UOP_STAT_INC(uopcode, miss);\n") + self.emit(f"SET_CURRENT_CACHED_VALUES({self.exit_cache_depth});\n") self.emit("JUMP_TO_JUMP_TARGET();\n") self.emit("}\n") return not always_true(first_tkn) - periodic_if = deopt_if + periodic_if = deopt_if = exit_if def oparg( self, @@ -135,8 +122,47 @@ def oparg( return True -def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: + def tier2_to_tier2( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + assert self.exit_cache_depth == 0, uop.name + cache_items(self, storage.stack, self.exit_cache_depth, False) + storage.flush(self.out) + self.out.emit(tkn) + lparen = next(tkn_iter) + assert lparen.kind == "LPAREN" + self.emit(lparen) + emit_to(self.out, tkn_iter, "RPAREN") + self.out.emit(")") + return False + + goto_tier_one = tier2_to_tier2 + +def cache_items(emitter: Emitter, stack: Stack, cached_items: int, zero_regs: bool) -> None: + emitter.out.start_line() + i = cached_items + while i > 0: + emitter.out.start_line() + item = StackItem(f"_tos_cache{i-1}", "", False, True) + stack.pop(item, emitter.out) + i -= 1 + if zero_regs: + # TO DO -- For compilers that support it, + # replace this with a "clobber" to tell + # the compiler that these values are unused + # without having to emit any code. + for i in range(cached_items, MAX_CACHED_REGISTER): + emitter.out.emit(f"_tos_cache{i} = PyStackRef_ZERO_BITS;\n") + emitter.emit(f"SET_CURRENT_CACHED_VALUES({cached_items});\n") + +def write_uop(uop: Uop, emitter: Emitter, stack: Stack, cached_items: int = 0) -> tuple[bool, Stack]: locals: dict[str, Local] = {} + zero_regs = is_large(uop) or uop.properties.escapes try: emitter.out.start_line() if uop.properties.oparg: @@ -156,14 +182,22 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: cast = f"uint{cache.size*16}_t" emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n") idx += 1 - _, storage = emitter.emit_tokens(uop, storage, None, False) - storage.flush(emitter.out) + reachable, storage = emitter.emit_tokens(uop, storage, None, False) + if reachable: + storage.stack._print(emitter.out) + cache_items(emitter, storage.stack, cached_items, zero_regs) + storage.flush(emitter.out) + return reachable, storage.stack except StackError as ex: raise analysis_error(ex.args[0], uop.body.open) from None - return storage.stack SKIPS = ("_EXTENDED_ARG",) +def is_for_iter_test(uop: Uop) -> bool: + return uop.name in ( + "_GUARD_NOT_EXHAUSTED_RANGE", "_GUARD_NOT_EXHAUSTED_LIST", + "_GUARD_NOT_EXHAUSTED_TUPLE", "_FOR_ITER_TIER_TWO" + ) def generate_tier2( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool @@ -178,7 +212,7 @@ def generate_tier2( """ ) out = CWriter(outfile, 2, lines) - emitter = Tier2Emitter(out, analysis.labels) + out.emit("\n") for name, uop in analysis.uops.items(): if uop.properties.tier == 1: @@ -191,16 +225,24 @@ def generate_tier2( f"/* {uop.name} is not a viable micro-op for tier 2 because it {why_not_viable} */\n\n" ) continue - out.emit(f"case {uop.name}: {{\n") - declare_variables(uop, out) - stack = Stack() - stack = write_uop(uop, emitter, stack) - out.start_line() - if not uop.properties.always_exits: - out.emit("break;\n") - out.start_line() - out.emit("}") - out.emit("\n\n") + for inputs, outputs, exit_depth in get_uop_cache_depths(uop): + emitter = Tier2Emitter(out, analysis.labels, exit_depth) + out.emit(f"case {uop.name}_r{inputs}{outputs}: {{\n") + out.emit(f"CHECK_CURRENT_CACHED_VALUES({inputs});\n") + out.emit("assert(WITHIN_STACK_BOUNDS_WITH_CACHE());\n") + declare_variables(uop, out) + stack = Stack() + stack.push_cache([f"_tos_cache{i}" for i in range(inputs)], out) + stack._print(out) + reachable, stack = write_uop(uop, emitter, stack, outputs) + out.start_line() + if reachable: + out.emit("assert(WITHIN_STACK_BOUNDS_WITH_CACHE());\n") + if not uop.properties.always_exits: + out.emit("break;\n") + out.start_line() + out.emit("}") + out.emit("\n\n") outfile.write("#undef TIER_TWO\n") diff --git a/Tools/cases_generator/uop_id_generator.py b/Tools/cases_generator/uop_id_generator.py index aae89faaa928e1..485d431bc1bac5 100644 --- a/Tools/cases_generator/uop_id_generator.py +++ b/Tools/cases_generator/uop_id_generator.py @@ -8,6 +8,7 @@ from analyzer import ( Analysis, analyze_files, + get_uop_cache_depths, ) from generators_common import ( DEFAULT_INPUT, @@ -38,17 +39,21 @@ def generate_uop_ids( uops = [(uop.name, uop) for uop in analysis.uops.values()] # Sort so that _BASE comes immediately before _BASE_0, etc. for name, uop in sorted(uops): - if name in PRE_DEFINED: - continue - if uop.properties.tier == 1: + if name in PRE_DEFINED or uop.is_super() or uop.properties.tier == 1: continue if uop.implicitly_created and not distinct_namespace and not uop.replicated: out.emit(f"#define {name} {name[1:]}\n") else: out.emit(f"#define {name} {next_id}\n") next_id += 1 - out.emit(f"#define MAX_UOP_ID {next_id-1}\n") + for name, uop in sorted(uops): + if uop.properties.tier == 1: + continue + for inputs, outputs, _ in sorted(get_uop_cache_depths(uop)): + out.emit(f"#define {name}_r{inputs}{outputs} {next_id}\n") + next_id += 1 + out.emit(f"#define MAX_UOP_REGS_ID {next_id-1}\n") arg_parser = argparse.ArgumentParser( diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 1cc23837a72dea..011c16d5ec0187 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -8,6 +8,9 @@ from analyzer import ( Analysis, analyze_files, + get_uop_cache_depths, + Uop, + MAX_CACHED_REGISTER, ) from generators_common import ( DEFAULT_INPUT, @@ -21,17 +24,56 @@ DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_metadata.h" +def uop_cache_info(uop: Uop) -> list[str] | None: + if uop.name == "_SPILL_OR_RELOAD": + return None + default = "{ -1, -1, -1 },\n" + table_size = MAX_CACHED_REGISTER + 1 + entries = [ default ] * table_size + low = MAX_CACHED_REGISTER+1 + high = -1 + defined = [ False ] * 4 + for inputs, outputs, exit_depth in get_uop_cache_depths(uop): + entries[inputs] = f"{{ {outputs}, {exit_depth}, {uop.name}_r{inputs}{outputs} }},\n" + if inputs < low: + low = inputs + if inputs > high: + high = inputs + best = [ str(low if i < low else (high if high < i else i)) for i in range(MAX_CACHED_REGISTER+1) ] + + return [ f".best = {{ {', '.join(best)} }},\n", ".entries = {\n", ] + entries + [ "},\n" ] + +CACHING_INFO_DECL = """ +typedef struct _pyuop_tos_cache_entry { + /* input depth is implicit in position */ + int8_t output; + int8_t exit; + uint16_t opcode; +} _PyUopTOSentry; + +typedef struct _PyUopCachingInfo { + uint8_t best[MAX_CACHED_REGISTER + 1]; + _PyUopTOSentry entries[MAX_CACHED_REGISTER + 1]; +} _PyUopCachingInfo; + +extern const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1]; +""" + def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: + out.emit(f"#define MAX_CACHED_REGISTER {MAX_CACHED_REGISTER}\n") out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") out.emit("typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange;\n") out.emit("extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1];\n") - out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") - out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") + out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1];\n\n") + out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n") + out.emit(CACHING_INFO_DECL) + out.emit(f"extern const uint16_t _PyUop_SpillsAndReloads[{MAX_CACHED_REGISTER+1}][{MAX_CACHED_REGISTER+1}];\n") + out.emit("extern const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1];\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): - if uop.is_viable() and uop.properties.tier != 1: + if uop.is_viable() and uop.properties.tier != 1 and not uop.is_super(): out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") out.emit("};\n\n") @@ -42,16 +84,40 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: out.emit(f"[{uop.name}] = {{ {uop.replicated.start}, {uop.replicated.stop} }},\n") out.emit("};\n\n") - out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {\n") + out.emit("const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = {\n") + for uop in analysis.uops.values(): + if uop.is_viable() and uop.properties.tier != 1 and not uop.is_super(): + info = uop_cache_info(uop) + if info is not None: + out.emit(f"[{uop.name}] = {{\n") + for line in info: + out.emit(line) + out.emit("},\n") + out.emit("};\n\n") + out.emit("const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = {\n"); + for uop in analysis.uops.values(): + if uop.is_viable() and uop.properties.tier != 1 and not uop.is_super(): + for inputs, outputs, _ in get_uop_cache_depths(uop): + out.emit(f"[{uop.name}_r{inputs}{outputs}] = {uop.name},\n") + out.emit("};\n\n") + out.emit(f"const uint16_t _PyUop_SpillsAndReloads[{MAX_CACHED_REGISTER+1}][{MAX_CACHED_REGISTER+1}] = {{\n") + for i in range(MAX_CACHED_REGISTER+1): + for j in range(MAX_CACHED_REGISTER+1): + if i != j: + out.emit(f"[{i}][{j}] = _SPILL_OR_RELOAD_r{i}{j},\n") + out.emit("};\n\n") + out.emit("const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = {\n") for uop in sorted(analysis.uops.values(), key=lambda t: t.name): - if uop.is_viable() and uop.properties.tier != 1: + if uop.is_viable() and uop.properties.tier != 1 and not uop.is_super(): out.emit(f'[{uop.name}] = "{uop.name}",\n') + for inputs, outputs, _ in get_uop_cache_depths(uop): + out.emit(f'[{uop.name}_r{inputs}{outputs}] = "{uop.name}_r{inputs}{outputs}",\n') out.emit("};\n") out.emit("int _PyUop_num_popped(int opcode, int oparg)\n{\n") out.emit("switch(opcode) {\n") null = CWriter.null() for uop in analysis.uops.values(): - if uop.is_viable() and uop.properties.tier != 1: + if uop.is_viable() and uop.properties.tier != 1 and not uop.is_super(): stack = Stack() for var in reversed(uop.stack.inputs): if var.peek: diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 4f373011ebf079..59af7802544a01 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -24,7 +24,7 @@ def _dump_footer( yield "" yield f"static const StencilGroup trampoline = {groups['trampoline'].as_c('trampoline')};" yield "" - yield "static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {" + yield "static const StencilGroup stencil_groups[MAX_UOP_REGS_ID + 1] = {" for opname, group in sorted(groups.items()): if opname == "trampoline": continue diff --git a/Tools/jit/jit.h b/Tools/jit/jit.h index 10829654eabb38..d5cf288c660f00 100644 --- a/Tools/jit/jit.h +++ b/Tools/jit/jit.h @@ -9,4 +9,5 @@ typedef jit_func __attribute__((preserve_none)) jit_func_preserve_none; #define DECLARE_TARGET(NAME) \ _Py_CODEUNIT *__attribute__((preserve_none, visibility("hidden"))) \ - NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); + NAME(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, \ + _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2); diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 8f71010a1aff58..5d362c00969eab 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -52,7 +52,7 @@ do { \ OPT_STAT_INC(traces_executed); \ _PyExecutorObject *_executor = (EXECUTOR); \ jit_func_preserve_none jitted = _executor->jit_code; \ - __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ + __attribute__((musttail)) return jitted(frame, stack_pointer, tstate, _tos_cache0, _tos_cache1, _tos_cache2); \ } while (0) #undef GOTO_TIER_ONE @@ -76,7 +76,7 @@ do { \ #define PATCH_JUMP(ALIAS) \ do { \ DECLARE_TARGET(ALIAS); \ - __attribute__((musttail)) return ALIAS(frame, stack_pointer, tstate); \ + __attribute__((musttail)) return ALIAS(frame, stack_pointer, tstate, _tos_cache0, _tos_cache1, _tos_cache2); \ } while (0) #undef JUMP_TO_JUMP_TARGET @@ -88,8 +88,10 @@ do { \ #define TIER_TWO 2 __attribute__((preserve_none)) _Py_CODEUNIT * -_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) -{ +_JIT_ENTRY( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, + _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2 +) { // Locals that the instruction implementations expect to exist: PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) int oparg; diff --git a/Tools/jit/trampoline.c b/Tools/jit/trampoline.c index 79d6af97961fc9..f008a3d3507e72 100644 --- a/Tools/jit/trampoline.c +++ b/Tools/jit/trampoline.c @@ -10,7 +10,7 @@ _Py_CODEUNIT * _JIT_ENTRY( _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate ) { - typedef DECLARE_TARGET((*jit_func)); - jit_func jitted = (jit_func)exec->jit_code; - return jitted(frame, stack_pointer, tstate); + tstate->jit_exit = NULL; + jit_func_preserve_none jitted = (jit_func_preserve_none)exec->jit_code; + return jitted(frame, stack_pointer, tstate, PyStackRef_ZERO_BITS, PyStackRef_ZERO_BITS, PyStackRef_ZERO_BITS); }