diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 3344ede5e92c07..2a6dabf4f97c70 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -984,6 +984,7 @@ enum InstructionFormat { #define HAS_PASSTHROUGH_FLAG (4096) #define HAS_OPARG_AND_1_FLAG (8192) #define HAS_ERROR_NO_POP_FLAG (16384) +#define HAS_STATIC_FLAG (32768) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -999,6 +1000,7 @@ enum InstructionFormat { #define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) #define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) +#define OPCODE_HAS_STATIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_STATIC_FLAG)) #define OPARG_FULL 0 #define OPARG_CACHE_1 1 @@ -1091,7 +1093,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [END_FOR] = { true, INSTR_FMT_IX, 0 }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1155,7 +1157,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG }, [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, + [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, @@ -1177,13 +1179,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, - [NOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [NOP] = { true, INSTR_FMT_IX, 0 }, [POP_EXCEPT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [POP_JUMP_IF_FALSE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [POP_TOP] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1237,11 +1239,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_TRUE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, - [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, - [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, - [SETUP_CLEANUP] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [SETUP_FINALLY] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, - [SETUP_WITH] = { true, -1, HAS_PURE_FLAG | HAS_ARG_FLAG }, + [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, + [POP_BLOCK] = { true, -1, 0 }, + [SETUP_CLEANUP] = { true, -1, HAS_ARG_FLAG }, + [SETUP_FINALLY] = { true, -1, HAS_ARG_FLAG }, + [SETUP_WITH] = { true, -1, HAS_ARG_FLAG }, [STORE_FAST_MAYBE_NULL] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, }; #endif diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index f92c0a0cddf906..857f9c48555ac5 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -160,10 +160,13 @@ struct _Py_UopsSymbol { PyTypeObject *typ; // Borrowed reference PyObject *const_val; // Owned reference (!) unsigned int type_version; // currently stores type version + int locals_idx; + char is_static; // used for binding-time analysis }; #define UOP_FORMAT_TARGET 0 #define UOP_FORMAT_JUMP 1 +void _PyUOpPrint(const _PyUOpInstruction*); static inline uint32_t uop_get_target(const _PyUOpInstruction *inst) { @@ -199,14 +202,21 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) typedef struct _Py_UopsSymbol _Py_UopsSymbol; +typedef struct _Py_UopsLocalsPlusSlot { + _Py_UopsSymbol *sym; + char is_virtual; +} _Py_UopsLocalsPlusSlot; + struct _Py_UOpsAbstractFrame { // Max stacklen int stack_len; int locals_len; - _Py_UopsSymbol **stack_pointer; - _Py_UopsSymbol **stack; - _Py_UopsSymbol **locals; + _Py_UopsLocalsPlusSlot *stack_pointer; + _Py_UopsLocalsPlusSlot *stack; + _Py_UopsLocalsPlusSlot *locals; + + void *instr_ptr; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; @@ -229,34 +239,39 @@ struct _Py_UOpsContext { // Arena for the symbolic types. ty_arena t_arena; - _Py_UopsSymbol **n_consumed; - _Py_UopsSymbol **limit; - _Py_UopsSymbol *locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + _Py_UopsLocalsPlusSlot *n_consumed; + _Py_UopsLocalsPlusSlot *limit; + _Py_UopsLocalsPlusSlot locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; + + _PyUOpInstruction *trace_dest; + int n_trace_dest; }; typedef struct _Py_UOpsContext _Py_UOpsContext; -extern bool _Py_uop_sym_is_null(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_is_not_null(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_is_const(_Py_UopsSymbol *sym); -extern PyObject *_Py_uop_sym_get_const(_Py_UopsSymbol *sym); -extern _Py_UopsSymbol *_Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); -extern _Py_UopsSymbol *_Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); -extern _Py_UopsSymbol *_Py_uop_sym_new_type( +extern bool _Py_uop_sym_is_null(_Py_UopsLocalsPlusSlot sym); +extern bool _Py_uop_sym_is_not_null(_Py_UopsLocalsPlusSlot sym); +extern bool _Py_uop_sym_is_const(_Py_UopsLocalsPlusSlot sym); +extern PyObject *_Py_uop_sym_get_const(_Py_UopsLocalsPlusSlot sym); +extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_unknown(_Py_UOpsContext *ctx); +extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_not_null(_Py_UOpsContext *ctx); +extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_type( _Py_UOpsContext *ctx, PyTypeObject *typ); -extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); -extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); -extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym); -extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); -extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); -extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ); -extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version); -extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val); -extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); -extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym); -extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsSymbol *sym); +extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); +extern _Py_UopsLocalsPlusSlot _Py_uop_sym_new_null(_Py_UOpsContext *ctx); +extern bool _Py_uop_sym_has_type(_Py_UopsLocalsPlusSlot sym); +extern bool _Py_uop_sym_matches_type(_Py_UopsLocalsPlusSlot sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(_Py_UopsLocalsPlusSlot sym, unsigned int version); +extern void _Py_uop_sym_set_locals_idx(_Py_UopsLocalsPlusSlot sym, int locals_idx); +extern int _Py_uop_sym_get_locals_idx(_Py_UopsLocalsPlusSlot sym); +extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym); +extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym); +extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, unsigned int version); +extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyObject *const_val); +extern bool _Py_uop_sym_is_bottom(_Py_UopsLocalsPlusSlot sym); +extern int _Py_uop_sym_truthiness(_Py_UopsLocalsPlusSlot sym); +extern PyTypeObject *_Py_uop_sym_get_type(_Py_UopsLocalsPlusSlot sym); extern void _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); @@ -266,7 +281,7 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( _Py_UOpsContext *ctx, PyCodeObject *co, int curr_stackentries, - _Py_UopsSymbol **args, + _Py_UopsLocalsPlusSlot *args, int arg_len); extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 07606135d7a356..d6e977000e17f5 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -19,35 +19,35 @@ extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { - [_NOP] = HAS_PURE_FLAG, + [_NOP] = HAS_STATIC_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, - [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_3] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_4] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_5] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_6] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST_7] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_STATIC_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, + [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_STATIC_FLAG, + [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_STATIC_FLAG, [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_POP_TOP] = HAS_PURE_FLAG, + [_POP_TOP] = HAS_STATIC_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_SEND] = HAS_PURE_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -265,8 +265,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, - [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, + [_LOAD_CONST_INLINE] = HAS_STATIC_FLAG, + [_LOAD_CONST_INLINE_BORROW] = HAS_STATIC_FLAG, [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_WITH_NULL] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG, diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f1ab72180d714d..302c214b74b28b 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1481,6 +1481,33 @@ def fn(a): fn(A()) + def test_pe_load_fast_pop_top(self): + def thing(a): + x = 0 + for i in range(20): + i + return i + + + res, ex = self._run_with_optimizer(thing, 1) + self.assertEqual(res, 19) + self.assertIsNotNone(ex) + self.assertEqual(list(iter_opnames(ex)).count("_POP_TOP"), 0) + self.assertTrue(ex.is_valid()) + + def test_pe_dead_store_elimination(self): + def thing(a): + x = 0 + for i in range(20): + x = x + return i + + + res, ex = self._run_with_optimizer(thing, 1) + self.assertEqual(res, 19) + self.assertIsNotNone(ex) + self.assertEqual(list(iter_opnames(ex)).count("_LOAD_FAST_1"), 0) + self.assertTrue(ex.is_valid()) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 214e53dde64bbf..e8d9d1861adf3a 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1264,15 +1264,15 @@ def test_overridden_abstract_args(self): """ output = """ case OP: { - _Py_UopsSymbol *arg1; - _Py_UopsSymbol *out; + _Py_UopsLocalsPlusSlot arg1; + _Py_UopsLocalsPlusSlot out; eggs(); stack_pointer[-1] = out; break; } case OP2: { - _Py_UopsSymbol *out; + _Py_UopsLocalsPlusSlot out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; @@ -1296,15 +1296,15 @@ def test_no_overridden_case(self): """ output = """ case OP: { - _Py_UopsSymbol *out; + _Py_UopsLocalsPlusSlot out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } case OP2: { - _Py_UopsSymbol *arg1; - _Py_UopsSymbol *out; + _Py_UopsLocalsPlusSlot arg1; + _Py_UopsLocalsPlusSlot out; stack_pointer[-1] = out; break; } diff --git a/Makefile.pre.in b/Makefile.pre.in index 07c8a4d20142db..028a0272ef4d28 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1958,7 +1958,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ regen-generated-cases regen-executor-cases regen-optimizer-cases \ - regen-opcode-metadata regen-uop-metadata + regen-partial-evaluator-cases regen-opcode-metadata regen-uop-metadata .PHONY: regen-opcode-ids regen-opcode-ids: @@ -2004,6 +2004,16 @@ regen-optimizer-cases: $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new +.PHONY: regen-partial-evaluator-cases +regen-partial-evaluator-cases: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/optimizer_generator.py \ + -o $(srcdir)/Python/partial_evaluator_cases.c.h.new \ + $(srcdir)/Python/optimizer_bytecodes.c \ + $(srcdir)/Python/partial_evaluator_bytecodes.c \ + $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/partial_evaluator_cases.c.h $(srcdir)/Python/partial_evaluator_cases.c.h.new + + .PHONY: regen-opcode-metadata regen-opcode-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ @@ -2041,7 +2051,8 @@ Python/optimizer.o: \ Python/optimizer_analysis.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Include/internal/pycore_optimizer.h \ - $(srcdir)/Python/optimizer_cases.c.h + $(srcdir)/Python/optimizer_cases.c.h \ + $(srcdir)/Python/partial_evaluator_cases.c.h Python/frozen.o: $(FROZEN_FILES_OUT) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst b/Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst new file mode 100644 index 00000000000000..6029db0998ebe9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-09-04-03-36-48.gh-issue-120619.yE7lQb.rst @@ -0,0 +1 @@ +Set up a tier 2 partial evaluation pass. Patch by Ken Jin. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c712c772201e10..eef07a7e734f7a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -140,7 +140,7 @@ dummy_func( switch (opcode) { // BEGIN BYTECODES // - pure inst(NOP, (--)) { + _static inst(NOP, (--)) { } family(RESUME, 0) = { @@ -239,7 +239,7 @@ dummy_func( value = PyStackRef_DUP(value_s); } - replicate(8) pure inst(LOAD_FAST, (-- value)) { + replicate(8) _static inst(LOAD_FAST, (-- value)) { assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); } @@ -261,7 +261,7 @@ dummy_func( value = PyStackRef_FromPyObjectNew(GETITEM(FRAME_CO_CONSTS, oparg)); } - replicate(8) inst(STORE_FAST, (value --)) { + replicate(8) _static inst(STORE_FAST, (value --)) { SETLOCAL(oparg, value); } @@ -283,7 +283,7 @@ dummy_func( SETLOCAL(oparg2, value2); } - pure inst(POP_TOP, (value --)) { + _static inst(POP_TOP, (value --)) { DECREF_INPUTS(); } @@ -4704,7 +4704,7 @@ dummy_func( if (lltrace >= 2) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %ld, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); @@ -4749,11 +4749,11 @@ dummy_func( DEOPT_IF(!current_executor->vm_data.valid); } - tier2 pure op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + tier2 _static op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectNew(ptr); } - tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + tier2 _static op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { value = PyStackRef_FromPyObjectImmortal(ptr); } @@ -4793,7 +4793,7 @@ dummy_func( if (lltrace >= 2) { printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %ld, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index fdfec66b73c730..a44946704f262e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5251,7 +5251,7 @@ if (lltrace >= 2) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %ld, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(code)), _PyOpcode_OpName[target->op.code]); @@ -5389,7 +5389,7 @@ if (lltrace >= 2) { printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %u, temp %d, target %d -> %s]\n", + printf(", exit %ld, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.as_counter, (int)(target - _PyCode_CODE(_PyFrame_GetCode(frame))), _PyOpcode_OpName[target->op.code]); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index f30e873605d858..0bf54f50d56d7b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -324,6 +324,8 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) #define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_set_locals_idx _Py_uop_sym_set_locals_idx +#define sym_get_locals_idx _Py_uop_sym_get_locals_idx #define sym_is_bottom _Py_uop_sym_is_bottom #define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new @@ -333,8 +335,8 @@ static int optimize_to_bool( _PyUOpInstruction *this_instr, _Py_UOpsContext *ctx, - _Py_UopsSymbol *value, - _Py_UopsSymbol **result_ptr) + _Py_UopsLocalsPlusSlot value, + _Py_UopsLocalsPlusSlot *result_ptr) { if (sym_matches_type(value, &PyBool_Type)) { REPLACE_OP(this_instr, _NOP, 0, 0); @@ -385,6 +387,12 @@ get_code(_PyUOpInstruction *op) return co; } +static inline _Py_UopsLocalsPlusSlot +sym_to_slot(_Py_UopsSymbol *sym) +{ + return (_Py_UopsLocalsPlusSlot){sym, 0}; +} + /* 1 for success, 0 for not ready, cannot error at the moment. */ static int optimize_uops( @@ -422,7 +430,7 @@ optimize_uops( int oparg = this_instr->oparg; opcode = this_instr->opcode; - _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; + _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -485,6 +493,191 @@ optimize_uops( } +#define WRITE_OP(INST, OP, ARG, OPERAND) \ + (INST)->opcode = OP; \ + (INST)->oparg = ARG; \ + (INST)->operand = OPERAND; + +#define SET_STATIC_INST() instr_is_truly_static = true; + +static void +reify_shadow_stack(_Py_UOpsContext *ctx) +{ + _PyUOpInstruction *trace_dest = ctx->trace_dest; + for (_Py_UopsLocalsPlusSlot *sp = ctx->frame->stack; sp < ctx->frame->stack_pointer; sp++) { + _Py_UopsLocalsPlusSlot slot = *sp; + assert(slot.sym != NULL); + // Need reifying. + if (slot.is_virtual) { + sp->is_virtual = false; + if (slot.sym->locals_idx >= 0) { + DPRINTF(3, "reifying %d LOAD_FAST %d\n", (int)(sp - ctx->frame->stack), slot.sym->locals_idx); + WRITE_OP(&trace_dest[ctx->n_trace_dest], _LOAD_FAST, slot.sym->locals_idx, 0); + trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; + trace_dest[ctx->n_trace_dest].target = 0; + } + else if (slot.sym->const_val) { + DPRINTF(3, "reifying %d LOAD_CONST_INLINE %p\n", (int)(sp - ctx->frame->stack), slot.sym->const_val); + WRITE_OP(&trace_dest[ctx->n_trace_dest], _Py_IsImmortal(slot.sym->const_val) ? + _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uint64_t)slot.sym->const_val); + trace_dest[ctx->n_trace_dest].format = UOP_FORMAT_TARGET; + trace_dest[ctx->n_trace_dest].target = 0; + } + else if (sym_is_null(slot)) { + DPRINTF(3, "reifying %d PUSH_NULL\n", (int)(sp - ctx->frame->stack)); + WRITE_OP(&trace_dest[ctx->n_trace_dest], _PUSH_NULL, 0, 0); + } + else { + // Is static but not a constant value of locals or NULL. + // How is that possible? + Py_UNREACHABLE(); + } + ctx->n_trace_dest++; + if (ctx->n_trace_dest >= UOP_MAX_TRACE_LENGTH) { + ctx->out_of_space = true; + ctx->done = true; + return; + } + } + } +} + +/* 1 for success, 0 for not ready, cannot error at the moment. */ +static int +partial_evaluate_uops( + PyCodeObject *co, + _PyUOpInstruction *trace, + int trace_len, + int curr_stacklen, + _PyBloomFilter *dependencies +) +{ + + _PyUOpInstruction trace_dest[UOP_MAX_TRACE_LENGTH]; + _Py_UOpsContext context; + context.trace_dest = trace_dest; + context.n_trace_dest = 0; + _Py_UOpsContext *ctx = &context; + uint32_t opcode = UINT16_MAX; + int curr_space = 0; + int max_space = 0; + _PyUOpInstruction *first_valid_check_stack = NULL; + _PyUOpInstruction *corresponding_check_stack = NULL; + + _Py_uop_abstractcontext_init(ctx); + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); + if (frame == NULL) { + return -1; + } + ctx->curr_frame_depth++; + ctx->frame = frame; + ctx->done = false; + ctx->out_of_space = false; + ctx->contradiction = false; + + _PyUOpInstruction *this_instr = NULL; + int i = 0; + for (; !ctx->done; i++) { + assert(i < trace_len); + this_instr = &trace[i]; + + int oparg = this_instr->oparg; + opcode = this_instr->opcode; + _Py_UopsLocalsPlusSlot *stack_pointer = ctx->frame->stack_pointer; + + // An instruction is candidate static if it has no escapes, and all its inputs + // are static. + // If so, whether it can be eliminated is up to whether it has an implementation. + bool instr_is_truly_static = false; + if (!(_PyUop_Flags[opcode] & HAS_STATIC_FLAG)) { + reify_shadow_stack(ctx); + } + +#ifdef Py_DEBUG + if (get_lltrace() >= 3) { + printf("%4d pe: ", (int)(this_instr - trace)); + _PyUOpPrint(this_instr); + printf(" "); + } +#endif + + switch (opcode) { + +#include "partial_evaluator_cases.c.h" + + default: + DPRINTF(1, "\nUnknown opcode in pe's abstract interpreter\n"); + Py_UNREACHABLE(); + } + assert(ctx->frame != NULL); + DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); + ctx->frame->stack_pointer = stack_pointer; + assert(STACK_LEVEL() >= 0); + if (!instr_is_truly_static) { + trace_dest[ctx->n_trace_dest] = *this_instr; + ctx->n_trace_dest++; + if (ctx->n_trace_dest >= UOP_MAX_TRACE_LENGTH) { + ctx->out_of_space = true; + ctx->done = true; + } + } + else { + // Inst is static. Nothing written :)! + assert((_PyUop_Flags[opcode] & HAS_STATIC_FLAG)); +#ifdef Py_DEBUG + if (get_lltrace() >= 3) { + printf("%4d pe -STATIC-\n", (int) (this_instr - trace)); + } +#endif + } + if (ctx->done) { + break; + } + } + if (ctx->out_of_space) { + DPRINTF(3, "\n"); + DPRINTF(1, "Out of space in pe's abstract interpreter\n"); + } + if (ctx->contradiction) { + // Attempted to push a "bottom" (contradiction) symbol onto the stack. + // This means that the abstract interpreter has hit unreachable code. + // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting + // bottom indicates type instability, so we are probably better off + // retrying later. + DPRINTF(3, "\n"); + DPRINTF(1, "Hit bottom in pe's abstract interpreter\n"); + _Py_uop_abstractcontext_fini(ctx); + return 0; + } + + if (ctx->out_of_space || !is_terminator(this_instr)) { + _Py_uop_abstractcontext_fini(ctx); + return trace_len; + } + else { + // We MUST not have bailed early here. + // That's the only time the PE's residual is valid. + assert(ctx->n_trace_dest < UOP_MAX_TRACE_LENGTH); + assert(is_terminator(this_instr)); + assert(ctx->n_trace_dest <= trace_len); + + // Copy trace_dest into trace. + memcpy(trace, trace_dest, ctx->n_trace_dest * sizeof(_PyUOpInstruction )); + int trace_dest_len = ctx->n_trace_dest; + _Py_uop_abstractcontext_fini(ctx); + return trace_dest_len; + } + +error: + DPRINTF(3, "\n"); + DPRINTF(1, "Encountered error in pe's abstract interpreter\n"); + if (opcode <= MAX_UOP_ID) { + OPT_ERROR_IN_OPCODE(opcode); + } + _Py_uop_abstractcontext_fini(ctx); + return -1; + +} static int remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) @@ -598,9 +791,20 @@ _Py_uop_analyze_and_optimize( return length; } + // Help the PE by removing as many _CHECK_VALIDITY as possible, + // Since PE treats that as non-static since it can deopt arbitrarily. length = remove_unneeded_uops(buffer, length); assert(length > 0); + length = partial_evaluate_uops( + _PyFrame_GetCode(frame), buffer, + length, curr_stacklen, dependencies); + + if (length <= 0) { + return length; + } + + OPT_STAT_INC(optimizer_successes); return length; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 9a1b9da52f4bb5..9877deadad2941 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -85,11 +85,11 @@ dummy_func(void) { op(_LOAD_FAST_AND_CLEAR, (-- value)) { value = GETLOCAL(oparg); - _Py_UopsSymbol *temp = sym_new_null(ctx); + _Py_UopsLocalsPlusSlot temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; } - op(_STORE_FAST, (value --)) { + _static op(_STORE_FAST, (value --)) { GETLOCAL(oparg) = value; } @@ -329,10 +329,10 @@ dummy_func(void) { } } - op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame)) { (void)container; (void)sub; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; } @@ -487,7 +487,7 @@ dummy_func(void) { op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { (void)index; null = sym_new_null(ctx); - attr = NULL; + attr = (_Py_UopsLocalsPlusSlot){NULL, 0}; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. assert(sym_is_const(owner)); @@ -500,7 +500,7 @@ dummy_func(void) { attr = sym_new_const(ctx, res); } } - if (attr == NULL) { + if (attr.sym == NULL) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -545,10 +545,10 @@ dummy_func(void) { self = owner; } - op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame)) { (void)fget; (void)owner; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; } @@ -568,7 +568,7 @@ dummy_func(void) { sym_set_type(callable, &PyMethod_Type); } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { int argcount = oparg; (void)callable; @@ -594,7 +594,7 @@ dummy_func(void) { DPRINTF(3, "code=%p ", co); } - assert(self_or_null != NULL); + assert(self_or_null.sym != NULL); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -603,9 +603,9 @@ dummy_func(void) { } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = frame_new(ctx, co, 0, args, argcount); + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, args, argcount); } else { - new_frame = frame_new(ctx, co, 0, NULL, 0); + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, NULL, 0); } } @@ -618,21 +618,21 @@ dummy_func(void) { maybe_self = sym_new_not_null(ctx); } - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ (void)callable; (void)self_or_null; (void)args; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { (void)callable; (void)self_or_null; (void)args; (void)kwnames; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; } @@ -645,11 +645,11 @@ dummy_func(void) { init = sym_new_not_null(ctx); } - op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) { + op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame)) { (void)self; (void)init; (void)args; - init_frame = NULL; + init_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; } @@ -723,12 +723,12 @@ dummy_func(void) { Py_UNREACHABLE(); } - op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { + op(_PUSH_FRAME, (new_frame -- unused if (0))) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)new_frame.sym; ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; + stack_pointer = ((_Py_UOpsAbstractFrame *)new_frame.sym)->stack_pointer; co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4d172e3c762704..3d180994f2954c 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -24,7 +24,7 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ case _LOAD_FAST_CHECK: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); // We guarantee this will error - just bail and don't optimize it. if (sym_is_null(value)) { @@ -37,7 +37,7 @@ } case _LOAD_FAST: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; @@ -46,9 +46,9 @@ } case _LOAD_FAST_AND_CLEAR: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = GETLOCAL(oparg); - _Py_UopsSymbol *temp = sym_new_null(ctx); + _Py_UopsLocalsPlusSlot temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; @@ -57,7 +57,7 @@ } case _LOAD_CONST: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; PyObject *val = PyTuple_GET_ITEM(co->co_consts, this_instr->oparg); int opcode = _Py_IsImmortal(val) ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE; REPLACE_OP(this_instr, opcode, 0, (uintptr_t)val); @@ -69,7 +69,7 @@ } case _STORE_FAST: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; @@ -84,7 +84,7 @@ } case _PUSH_NULL: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -93,7 +93,7 @@ } case _END_SEND: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; @@ -102,22 +102,22 @@ } case _UNARY_NEGATIVE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_NOT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _TO_BOOL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { res = sym_new_type(ctx, &PyBool_Type); @@ -127,8 +127,8 @@ } case _TO_BOOL_BOOL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyBool_Type); @@ -139,8 +139,8 @@ } case _TO_BOOL_INT: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyLong_Type); @@ -151,8 +151,8 @@ } case _TO_BOOL_LIST: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_type(value, &PyList_Type); @@ -163,8 +163,8 @@ } case _TO_BOOL_NONE: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { sym_set_const(value, Py_None); @@ -175,8 +175,8 @@ } case _TO_BOOL_STR: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; value = stack_pointer[-1]; if (!optimize_to_bool(this_instr, ctx, value, &res)) { res = sym_new_type(ctx, &PyBool_Type); @@ -187,22 +187,22 @@ } case _REPLACE_WITH_TRUE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _UNARY_INVERT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _GUARD_BOTH_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyLong_Type)) { @@ -232,9 +232,9 @@ } case _BINARY_OP_MULTIPLY_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -262,9 +262,9 @@ } case _BINARY_OP_ADD_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -292,9 +292,9 @@ } case _BINARY_OP_SUBTRACT_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -322,8 +322,8 @@ } case _GUARD_BOTH_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyFloat_Type)) { @@ -353,9 +353,9 @@ } case _BINARY_OP_MULTIPLY_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -384,9 +384,9 @@ } case _BINARY_OP_ADD_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -415,9 +415,9 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -446,8 +446,8 @@ } case _GUARD_BOTH_UNICODE: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_matches_type(left, &PyUnicode_Type) && @@ -460,9 +460,9 @@ } case _BINARY_OP_ADD_UNICODE: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; if (sym_is_const(left) && sym_is_const(right) && @@ -490,7 +490,7 @@ } case _BINARY_SUBSCR: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -499,7 +499,7 @@ } case _BINARY_SLICE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -514,7 +514,7 @@ } case _BINARY_SUBSCR_LIST_INT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -523,7 +523,7 @@ } case _BINARY_SUBSCR_STR_INT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -532,7 +532,7 @@ } case _BINARY_SUBSCR_TUPLE_INT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -541,7 +541,7 @@ } case _BINARY_SUBSCR_DICT: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -554,16 +554,16 @@ } case _BINARY_SUBSCR_INIT_CALL: { - _Py_UopsSymbol *sub; - _Py_UopsSymbol *container; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot sub; + _Py_UopsLocalsPlusSlot container; + _Py_UopsLocalsPlusSlot new_frame; sub = stack_pointer[-1]; container = stack_pointer[-2]; (void)container; (void)sub; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-2] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-2] = new_frame; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); break; @@ -606,14 +606,14 @@ } case _CALL_INTRINSIC_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _CALL_INTRINSIC_2: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -622,8 +622,8 @@ } case _RETURN_VALUE: { - _Py_UopsSymbol *retval; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot retval; + _Py_UopsLocalsPlusSlot res; retval = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -650,14 +650,14 @@ } case _GET_AITER: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - _Py_UopsSymbol *awaitable; + _Py_UopsLocalsPlusSlot awaitable; awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; @@ -666,7 +666,7 @@ } case _GET_AWAITABLE: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -681,7 +681,7 @@ } case _YIELD_VALUE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_unknown(ctx); stack_pointer[-1] = res; break; @@ -694,7 +694,7 @@ } case _LOAD_COMMON_CONSTANT: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -703,7 +703,7 @@ } case _LOAD_BUILD_CLASS: { - _Py_UopsSymbol *bc; + _Py_UopsLocalsPlusSlot bc; bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; @@ -722,8 +722,8 @@ } case _UNPACK_SEQUENCE: { - _Py_UopsSymbol *seq; - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot seq; + _Py_UopsLocalsPlusSlot *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ @@ -737,8 +737,8 @@ } case _UNPACK_SEQUENCE_TWO_TUPLE: { - _Py_UopsSymbol *val1; - _Py_UopsSymbol *val0; + _Py_UopsLocalsPlusSlot val1; + _Py_UopsLocalsPlusSlot val0; val1 = sym_new_not_null(ctx); val0 = sym_new_not_null(ctx); stack_pointer[-1] = val1; @@ -749,7 +749,7 @@ } case _UNPACK_SEQUENCE_TUPLE: { - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -760,7 +760,7 @@ } case _UNPACK_SEQUENCE_LIST: { - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot *values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); @@ -771,8 +771,8 @@ } case _UNPACK_EX: { - _Py_UopsSymbol *seq; - _Py_UopsSymbol **values; + _Py_UopsLocalsPlusSlot seq; + _Py_UopsLocalsPlusSlot *values; seq = stack_pointer[-1]; values = &stack_pointer[-1]; /* This has to be done manually */ @@ -809,7 +809,7 @@ } case _LOAD_LOCALS: { - _Py_UopsSymbol *locals; + _Py_UopsLocalsPlusSlot locals; locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; @@ -820,7 +820,7 @@ /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ case _LOAD_NAME: { - _Py_UopsSymbol *v; + _Py_UopsLocalsPlusSlot v; v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; @@ -829,8 +829,8 @@ } case _LOAD_GLOBAL: { - _Py_UopsSymbol **res; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot *res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; res = &stack_pointer[0]; for (int _i = 1; --_i >= 0;) { res[_i] = sym_new_not_null(ctx); @@ -851,8 +851,8 @@ } case _LOAD_GLOBAL_MODULE: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -863,8 +863,8 @@ } case _LOAD_GLOBAL_BUILTINS: { - _Py_UopsSymbol *res; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; res = sym_new_not_null(ctx); null = sym_new_null(ctx); stack_pointer[0] = res; @@ -887,14 +887,14 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_DEREF: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; @@ -913,7 +913,7 @@ } case _BUILD_STRING: { - _Py_UopsSymbol *str; + _Py_UopsLocalsPlusSlot str; str = sym_new_not_null(ctx); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -922,7 +922,7 @@ } case _BUILD_TUPLE: { - _Py_UopsSymbol *tup; + _Py_UopsLocalsPlusSlot tup; tup = sym_new_not_null(ctx); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -931,7 +931,7 @@ } case _BUILD_LIST: { - _Py_UopsSymbol *list; + _Py_UopsLocalsPlusSlot list; list = sym_new_not_null(ctx); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -952,7 +952,7 @@ } case _BUILD_SET: { - _Py_UopsSymbol *set; + _Py_UopsLocalsPlusSlot set; set = sym_new_not_null(ctx); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -961,7 +961,7 @@ } case _BUILD_MAP: { - _Py_UopsSymbol *map; + _Py_UopsLocalsPlusSlot map; map = sym_new_not_null(ctx); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -994,7 +994,7 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ case _LOAD_SUPER_ATTR_ATTR: { - _Py_UopsSymbol *attr_st; + _Py_UopsLocalsPlusSlot attr_st; attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; @@ -1003,8 +1003,8 @@ } case _LOAD_SUPER_ATTR_METHOD: { - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); stack_pointer[-3] = attr; @@ -1015,9 +1015,9 @@ } case _LOAD_ATTR: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); @@ -1032,7 +1032,7 @@ } case _GUARD_TYPE_VERSION: { - _Py_UopsSymbol *owner; + _Py_UopsLocalsPlusSlot owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)this_instr->operand; assert(type_version); @@ -1062,9 +1062,9 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t offset = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1079,7 +1079,7 @@ } case _CHECK_ATTR_MODULE: { - _Py_UopsSymbol *owner; + _Py_UopsLocalsPlusSlot owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)this_instr->operand; (void)dict_version; @@ -1100,14 +1100,14 @@ } case _LOAD_ATTR_MODULE: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; (void)index; null = sym_new_null(ctx); - attr = NULL; + attr = (_Py_UopsLocalsPlusSlot){NULL, 0}; if (this_instr[-1].opcode == _NOP) { // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. assert(sym_is_const(owner)); @@ -1120,7 +1120,7 @@ attr = sym_new_const(ctx, res); } } - if (attr == NULL) { + if (attr.sym == NULL) { /* No conversion made. We don't know what `attr` is. */ attr = sym_new_not_null(ctx); } @@ -1136,9 +1136,9 @@ } case _LOAD_ATTR_WITH_HINT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t hint = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1153,9 +1153,9 @@ } case _LOAD_ATTR_SLOT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1174,9 +1174,9 @@ } case _LOAD_ATTR_CLASS: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *null = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; attr = sym_new_not_null(ctx); @@ -1191,15 +1191,15 @@ } case _LOAD_ATTR_PROPERTY_FRAME: { - _Py_UopsSymbol *owner; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot new_frame; owner = stack_pointer[-1]; PyObject *fget = (PyObject *)this_instr->operand; (void)fget; (void)owner; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-1] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-1] = new_frame; break; } @@ -1228,9 +1228,9 @@ } case _COMPARE_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1248,9 +1248,9 @@ } case _COMPARE_OP_FLOAT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1263,9 +1263,9 @@ } case _COMPARE_OP_INT: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1278,9 +1278,9 @@ } case _COMPARE_OP_STR: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1293,9 +1293,9 @@ } case _IS_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1308,9 +1308,9 @@ } case _CONTAINS_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; (void)left; @@ -1323,7 +1323,7 @@ } case _CONTAINS_OP_SET: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -1332,7 +1332,7 @@ } case _CONTAINS_OP_DICT: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-2] = b; stack_pointer += -1; @@ -1341,8 +1341,8 @@ } case _CHECK_EG_MATCH: { - _Py_UopsSymbol *rest; - _Py_UopsSymbol *match; + _Py_UopsLocalsPlusSlot rest; + _Py_UopsLocalsPlusSlot match; rest = sym_new_not_null(ctx); match = sym_new_not_null(ctx); stack_pointer[-2] = rest; @@ -1351,14 +1351,14 @@ } case _CHECK_EXC_MATCH: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _IMPORT_NAME: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -1367,7 +1367,7 @@ } case _IMPORT_FROM: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1380,14 +1380,14 @@ /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { - _Py_UopsSymbol *b; + _Py_UopsLocalsPlusSlot b; b = sym_new_not_null(ctx); stack_pointer[-1] = b; break; } case _GET_LEN: { - _Py_UopsSymbol *len; + _Py_UopsLocalsPlusSlot len; len = sym_new_not_null(ctx); stack_pointer[0] = len; stack_pointer += 1; @@ -1396,7 +1396,7 @@ } case _MATCH_CLASS: { - _Py_UopsSymbol *attrs; + _Py_UopsLocalsPlusSlot attrs; attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1405,7 +1405,7 @@ } case _MATCH_MAPPING: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1414,7 +1414,7 @@ } case _MATCH_SEQUENCE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1423,7 +1423,7 @@ } case _MATCH_KEYS: { - _Py_UopsSymbol *values_or_none; + _Py_UopsLocalsPlusSlot values_or_none; values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1432,14 +1432,14 @@ } case _GET_ITER: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; } case _GET_YIELD_FROM_ITER: { - _Py_UopsSymbol *iter; + _Py_UopsLocalsPlusSlot iter; iter = sym_new_not_null(ctx); stack_pointer[-1] = iter; break; @@ -1448,7 +1448,7 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1469,7 +1469,7 @@ } case _ITER_NEXT_LIST: { - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1488,7 +1488,7 @@ } case _ITER_NEXT_TUPLE: { - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot next; next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; @@ -1507,8 +1507,8 @@ } case _ITER_NEXT_RANGE: { - _Py_UopsSymbol *iter; - _Py_UopsSymbol *next; + _Py_UopsLocalsPlusSlot iter; + _Py_UopsLocalsPlusSlot next; iter = stack_pointer[-1]; next = sym_new_type(ctx, &PyLong_Type); (void)iter; @@ -1525,9 +1525,9 @@ } case _LOAD_SPECIAL: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self_or_null; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null; owner = stack_pointer[-1]; (void)owner; attr = sym_new_not_null(ctx); @@ -1540,7 +1540,7 @@ } case _WITH_EXCEPT_START: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; @@ -1549,8 +1549,8 @@ } case _PUSH_EXC_INFO: { - _Py_UopsSymbol *prev_exc; - _Py_UopsSymbol *new_exc; + _Py_UopsLocalsPlusSlot prev_exc; + _Py_UopsLocalsPlusSlot new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); stack_pointer[-1] = prev_exc; @@ -1569,9 +1569,9 @@ } case _LOAD_ATTR_METHOD_WITH_VALUES: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1585,9 +1585,9 @@ } case _LOAD_ATTR_METHOD_NO_DICT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1601,14 +1601,14 @@ } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { - _Py_UopsSymbol *attr; + _Py_UopsLocalsPlusSlot attr; attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { - _Py_UopsSymbol *attr; + _Py_UopsLocalsPlusSlot attr; attr = sym_new_not_null(ctx); stack_pointer[-1] = attr; break; @@ -1619,9 +1619,9 @@ } case _LOAD_ATTR_METHOD_LAZY_DICT: { - _Py_UopsSymbol *owner; - _Py_UopsSymbol *attr; - _Py_UopsSymbol *self = NULL; + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; owner = stack_pointer[-1]; PyObject *descr = (PyObject *)this_instr->operand; (void)descr; @@ -1635,11 +1635,11 @@ } case _MAYBE_EXPAND_METHOD: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UopsSymbol *func; - _Py_UopsSymbol *maybe_self; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot func; + _Py_UopsLocalsPlusSlot maybe_self; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1659,10 +1659,10 @@ /* _MONITOR_CALL is not a viable micro-op for tier 2 */ case _PY_FRAME_GENERAL: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1670,9 +1670,9 @@ (void)callable; (void)self_or_null; (void)args; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -1687,8 +1687,8 @@ } case _EXPAND_METHOD: { - _Py_UopsSymbol *method; - _Py_UopsSymbol **self; + _Py_UopsLocalsPlusSlot method; + _Py_UopsLocalsPlusSlot *self; self = &stack_pointer[-1 - oparg]; method = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { @@ -1703,7 +1703,7 @@ } case _CALL_NON_PY_GENERAL: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1712,8 +1712,8 @@ } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsSymbol *null; - _Py_UopsSymbol *callable; + _Py_UopsLocalsPlusSlot null; + _Py_UopsLocalsPlusSlot callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_null(null); @@ -1722,9 +1722,9 @@ } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { - _Py_UopsSymbol *callable; - _Py_UopsSymbol *func; - _Py_UopsSymbol *self; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot func; + _Py_UopsLocalsPlusSlot self; callable = stack_pointer[-2 - oparg]; (void)callable; func = sym_new_not_null(ctx); @@ -1744,8 +1744,8 @@ } case _CHECK_FUNCTION_EXACT_ARGS: { - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; sym_set_type(callable, &PyFunction_Type); @@ -1760,10 +1760,10 @@ } case _INIT_CALL_PY_EXACT_ARGS: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1789,7 +1789,7 @@ co = (PyCodeObject *)func->func_code; DPRINTF(3, "code=%p ", co); } - assert(self_or_null != NULL); + assert(self_or_null.sym != NULL); assert(args != NULL); if (sym_is_not_null(self_or_null)) { // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM @@ -1797,25 +1797,25 @@ argcount++; } if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - new_frame = frame_new(ctx, co, 0, args, argcount); + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, args, argcount); } else { - new_frame = frame_new(ctx, co, 0, NULL, 0); + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, NULL, 0); } - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; } case _PUSH_FRAME: { - _Py_UOpsAbstractFrame *new_frame; - new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + _Py_UopsLocalsPlusSlot new_frame; + new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); ctx->frame->stack_pointer = stack_pointer; - ctx->frame = new_frame; + ctx->frame = (_Py_UOpsAbstractFrame *)new_frame.sym; ctx->curr_frame_depth++; - stack_pointer = new_frame->stack_pointer; + stack_pointer = ((_Py_UOpsAbstractFrame *)new_frame.sym)->stack_pointer; co = get_code(this_instr); if (co == NULL) { // should be about to _EXIT_TRACE anyway @@ -1844,7 +1844,7 @@ } case _CALL_TYPE_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1853,7 +1853,7 @@ } case _CALL_STR_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1862,7 +1862,7 @@ } case _CALL_TUPLE_1: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3] = res; stack_pointer += -2; @@ -1871,11 +1871,11 @@ } case _CHECK_AND_ALLOCATE_OBJECT: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *null; - _Py_UopsSymbol *callable; - _Py_UopsSymbol *self; - _Py_UopsSymbol *init; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot init; args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1893,19 +1893,19 @@ } case _CREATE_INIT_FRAME: { - _Py_UopsSymbol **args; - _Py_UopsSymbol *init; - _Py_UopsSymbol *self; - _Py_UOpsAbstractFrame *init_frame; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot init; + _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot init_frame; args = &stack_pointer[-oparg]; init = stack_pointer[-1 - oparg]; self = stack_pointer[-2 - oparg]; (void)self; (void)init; (void)args; - init_frame = NULL; + init_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)init_frame; + stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -1918,7 +1918,7 @@ } case _CALL_BUILTIN_CLASS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1927,7 +1927,7 @@ } case _CALL_BUILTIN_O: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1936,7 +1936,7 @@ } case _CALL_BUILTIN_FAST: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1945,7 +1945,7 @@ } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1954,7 +1954,7 @@ } case _CALL_LEN: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1963,7 +1963,7 @@ } case _CALL_ISINSTANCE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1978,7 +1978,7 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1987,7 +1987,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1996,7 +1996,7 @@ } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2005,7 +2005,7 @@ } case _CALL_METHOD_DESCRIPTOR_FAST: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -2018,11 +2018,11 @@ /* _DO_CALL_KW is not a viable micro-op for tier 2 */ case _PY_FRAME_KW: { - _Py_UopsSymbol *kwnames; - _Py_UopsSymbol **args; - _Py_UopsSymbol *self_or_null; - _Py_UopsSymbol *callable; - _Py_UOpsAbstractFrame *new_frame; + _Py_UopsLocalsPlusSlot kwnames; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; @@ -2031,9 +2031,9 @@ (void)self_or_null; (void)args; (void)kwnames; - new_frame = NULL; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; ctx->done = true; - stack_pointer[-3 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); break; @@ -2048,9 +2048,9 @@ } case _EXPAND_METHOD_KW: { - _Py_UopsSymbol *method; - _Py_UopsSymbol **self; - _Py_UopsSymbol *kwnames; + _Py_UopsLocalsPlusSlot method; + _Py_UopsLocalsPlusSlot *self; + _Py_UopsLocalsPlusSlot kwnames; self = &stack_pointer[-2 - oparg]; method = sym_new_not_null(ctx); for (int _i = 1; --_i >= 0;) { @@ -2067,7 +2067,7 @@ } case _CALL_KW_NON_PY: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; @@ -2080,14 +2080,14 @@ /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ case _MAKE_FUNCTION: { - _Py_UopsSymbol *func; + _Py_UopsLocalsPlusSlot func; func = sym_new_not_null(ctx); stack_pointer[-1] = func; break; } case _SET_FUNCTION_ATTRIBUTE: { - _Py_UopsSymbol *func_st; + _Py_UopsLocalsPlusSlot func_st; func_st = sym_new_not_null(ctx); stack_pointer[-2] = func_st; stack_pointer += -1; @@ -2096,7 +2096,7 @@ } case _RETURN_GENERATOR: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -2120,7 +2120,7 @@ } case _BUILD_SLICE: { - _Py_UopsSymbol *slice; + _Py_UopsLocalsPlusSlot slice; slice = sym_new_not_null(ctx); stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -2129,21 +2129,21 @@ } case _CONVERT_VALUE: { - _Py_UopsSymbol *result; + _Py_UopsLocalsPlusSlot result; result = sym_new_not_null(ctx); stack_pointer[-1] = result; break; } case _FORMAT_SIMPLE: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-1] = res; break; } case _FORMAT_WITH_SPEC: { - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot res; res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; @@ -2152,8 +2152,8 @@ } case _COPY: { - _Py_UopsSymbol *bottom; - _Py_UopsSymbol *top; + _Py_UopsLocalsPlusSlot bottom; + _Py_UopsLocalsPlusSlot top; bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; @@ -2164,9 +2164,9 @@ } case _BINARY_OP: { - _Py_UopsSymbol *right; - _Py_UopsSymbol *left; - _Py_UopsSymbol *res; + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; right = stack_pointer[-1]; left = stack_pointer[-2]; PyTypeObject *ltype = sym_get_type(left); @@ -2192,8 +2192,8 @@ } case _SWAP: { - _Py_UopsSymbol *top; - _Py_UopsSymbol *bottom; + _Py_UopsLocalsPlusSlot top; + _Py_UopsLocalsPlusSlot bottom; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; stack_pointer[-2 - (oparg-2)] = top; @@ -2218,7 +2218,7 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2231,7 +2231,7 @@ } case _GUARD_IS_FALSE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2244,7 +2244,7 @@ } case _GUARD_IS_NONE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2261,7 +2261,7 @@ } case _GUARD_IS_NOT_NONE_POP: { - _Py_UopsSymbol *flag; + _Py_UopsLocalsPlusSlot flag; flag = stack_pointer[-1]; if (sym_is_const(flag)) { PyObject *value = sym_get_const(flag); @@ -2311,7 +2311,7 @@ } case _LOAD_CONST_INLINE: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; @@ -2321,7 +2321,7 @@ } case _LOAD_CONST_INLINE_BORROW: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); stack_pointer[0] = value; @@ -2331,15 +2331,15 @@ } case _POP_TOP_LOAD_CONST_INLINE_BORROW: { - _Py_UopsSymbol *value; + _Py_UopsLocalsPlusSlot value; value = sym_new_not_null(ctx); stack_pointer[-1] = value; break; } case _LOAD_CONST_INLINE_WITH_NULL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *null; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot null; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); @@ -2351,8 +2351,8 @@ } case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { - _Py_UopsSymbol *value; - _Py_UopsSymbol *null; + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot null; PyObject *ptr = (PyObject *)this_instr->operand; value = sym_new_const(ctx, ptr); null = sym_new_null(ctx); diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 40cbf95e3d6d39..22145330dd3b64 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -55,12 +55,17 @@ static _Py_UopsSymbol NO_SPACE_SYMBOL = { .type_version = 0, }; -_Py_UopsSymbol * +static _Py_UopsLocalsPlusSlot NO_SPACE_SLOT = { + .sym = &NO_SPACE_SYMBOL, + .is_virtual = 0, +}; + +_Py_UopsLocalsPlusSlot out_of_space(_Py_UOpsContext *ctx) { ctx->done = true; ctx->out_of_space = true; - return &NO_SPACE_SYMBOL; + return NO_SPACE_SLOT; } static _Py_UopsSymbol * @@ -77,29 +82,32 @@ sym_new(_Py_UOpsContext *ctx) self->typ = NULL; self->const_val = NULL; self->type_version = 0; + self->is_static = false; + self->locals_idx = -1; return self; } static inline void -sym_set_flag(_Py_UopsSymbol *sym, int flag) +sym_set_flag(_Py_UopsLocalsPlusSlot sym, int flag) { - sym->flags |= flag; + sym.sym->flags |= flag; } static inline void -sym_set_bottom(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +sym_set_bottom(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) { sym_set_flag(sym, IS_NULL | NOT_NULL); - sym->typ = NULL; - Py_CLEAR(sym->const_val); + sym.sym->typ = NULL; + Py_CLEAR(sym.sym->const_val); ctx->done = true; ctx->contradiction = true; } bool -_Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) +_Py_uop_sym_is_bottom(_Py_UopsLocalsPlusSlot sym_l) { + _Py_UopsSymbol *sym = sym_l.sym; if ((sym->flags & IS_NULL) && (sym->flags & NOT_NULL)) { assert(sym->flags == (IS_NULL | NOT_NULL)); assert(sym->typ == NULL); @@ -110,96 +118,98 @@ _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym) } bool -_Py_uop_sym_is_not_null(_Py_UopsSymbol *sym) +_Py_uop_sym_is_not_null(_Py_UopsLocalsPlusSlot sym) { - return sym->flags == NOT_NULL; + return sym.sym->flags == NOT_NULL; } bool -_Py_uop_sym_is_null(_Py_UopsSymbol *sym) +_Py_uop_sym_is_null(_Py_UopsLocalsPlusSlot sym) { - return sym->flags == IS_NULL; + return sym.sym->flags == IS_NULL; } bool -_Py_uop_sym_is_const(_Py_UopsSymbol *sym) +_Py_uop_sym_is_const(_Py_UopsLocalsPlusSlot sym) { - return sym->const_val != NULL; + return sym.sym->const_val != NULL; } PyObject * -_Py_uop_sym_get_const(_Py_UopsSymbol *sym) +_Py_uop_sym_get_const(_Py_UopsLocalsPlusSlot sym) { - return sym->const_val; + return sym.sym->const_val; } void -_Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ) +_Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); - if (sym->flags & IS_NULL) { + if (sym.sym->flags & IS_NULL) { sym_set_bottom(ctx, sym); return; } - if (sym->typ != NULL) { - if (sym->typ != typ) { + if (sym.sym->typ != NULL) { + if (sym.sym->typ != typ) { sym_set_bottom(ctx, sym); return; } } else { sym_set_flag(sym, NOT_NULL); - sym->typ = typ; + sym.sym->typ = typ; } } bool -_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version) +_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, unsigned int version) { // if the type version was already set, then it must be different and we should set it to bottom - if (sym->type_version) { + if (sym.sym->type_version) { sym_set_bottom(ctx, sym); return false; } - sym->type_version = version; + sym.sym->type_version = version; return true; } void -_Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val) +_Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym, PyObject *const_val) { assert(const_val != NULL); - if (sym->flags & IS_NULL) { + if (sym.sym->flags & IS_NULL) { sym_set_bottom(ctx, sym); } PyTypeObject *typ = Py_TYPE(const_val); - if (sym->typ != NULL && sym->typ != typ) { + if (sym.sym->typ != NULL && sym.sym->typ != typ) { sym_set_bottom(ctx, sym); } - if (sym->const_val != NULL) { - if (sym->const_val != const_val) { + if (sym.sym->const_val != NULL) { + if (sym.sym->const_val != const_val) { // TODO: What if they're equal? sym_set_bottom(ctx, sym); } } else { sym_set_flag(sym, NOT_NULL); - sym->typ = typ; - sym->const_val = Py_NewRef(const_val); + sym.sym->typ = typ; + sym.sym->const_val = Py_NewRef(const_val); } + sym.sym->is_static = true; } void -_Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +_Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) { if (_Py_uop_sym_is_not_null(sym)) { sym_set_bottom(ctx, sym); } sym_set_flag(sym, IS_NULL); + sym.sym->is_static = true; } void -_Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) +_Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsLocalsPlusSlot sym) { if (_Py_uop_sym_is_null(sym)) { sym_set_bottom(ctx, sym); @@ -208,36 +218,37 @@ _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym) } -_Py_UopsSymbol * +_Py_UopsLocalsPlusSlot _Py_uop_sym_new_unknown(_Py_UOpsContext *ctx) { - return sym_new(ctx); + return (_Py_UopsLocalsPlusSlot){sym_new(ctx), 0}; } -_Py_UopsSymbol * +_Py_UopsLocalsPlusSlot _Py_uop_sym_new_not_null(_Py_UOpsContext *ctx) { - _Py_UopsSymbol *res = _Py_uop_sym_new_unknown(ctx); - if (res == NULL) { + _Py_UopsLocalsPlusSlot res = _Py_uop_sym_new_unknown(ctx); + if (res.sym == NULL) { return out_of_space(ctx); } sym_set_flag(res, NOT_NULL); return res; } -_Py_UopsSymbol * +_Py_UopsLocalsPlusSlot _Py_uop_sym_new_type(_Py_UOpsContext *ctx, PyTypeObject *typ) { _Py_UopsSymbol *res = sym_new(ctx); if (res == NULL) { return out_of_space(ctx); } - _Py_uop_sym_set_type(ctx, res, typ); - return res; + _Py_UopsLocalsPlusSlot sym = {res, 0}; + _Py_uop_sym_set_type(ctx, sym, typ); + return sym; } // Adds a new reference to const_val, owned by the symbol. -_Py_UopsSymbol * +_Py_UopsLocalsPlusSlot _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) { assert(const_val != NULL); @@ -245,15 +256,16 @@ _Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val) if (res == NULL) { return out_of_space(ctx); } - _Py_uop_sym_set_const(ctx, res, const_val); - return res; + _Py_UopsLocalsPlusSlot sym = {res, 0}; + _Py_uop_sym_set_const(ctx, sym, const_val); + return sym; } -_Py_UopsSymbol * +_Py_UopsLocalsPlusSlot _Py_uop_sym_new_null(_Py_UOpsContext *ctx) { - _Py_UopsSymbol *null_sym = _Py_uop_sym_new_unknown(ctx); - if (null_sym == NULL) { + _Py_UopsLocalsPlusSlot null_sym = _Py_uop_sym_new_unknown(ctx); + if (null_sym.sym == NULL) { return out_of_space(ctx); } _Py_uop_sym_set_null(ctx, null_sym); @@ -261,45 +273,57 @@ _Py_uop_sym_new_null(_Py_UOpsContext *ctx) } PyTypeObject * -_Py_uop_sym_get_type(_Py_UopsSymbol *sym) +_Py_uop_sym_get_type(_Py_UopsLocalsPlusSlot sym) { if (_Py_uop_sym_is_bottom(sym)) { return NULL; } - return sym->typ; + return sym.sym->typ; } unsigned int -_Py_uop_sym_get_type_version(_Py_UopsSymbol *sym) +_Py_uop_sym_get_type_version(_Py_UopsLocalsPlusSlot sym) { - return sym->type_version; + return sym.sym->type_version; } bool -_Py_uop_sym_has_type(_Py_UopsSymbol *sym) +_Py_uop_sym_has_type(_Py_UopsLocalsPlusSlot sym) { if (_Py_uop_sym_is_bottom(sym)) { return false; } - return sym->typ != NULL; + return sym.sym->typ != NULL; } bool -_Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) +_Py_uop_sym_matches_type(_Py_UopsLocalsPlusSlot sym, PyTypeObject *typ) { assert(typ != NULL && PyType_Check(typ)); return _Py_uop_sym_get_type(sym) == typ; } bool -_Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version) +_Py_uop_sym_matches_type_version(_Py_UopsLocalsPlusSlot sym, unsigned int version) { return _Py_uop_sym_get_type_version(sym) == version; } +void +_Py_uop_sym_set_locals_idx(_Py_UopsLocalsPlusSlot sym, int locals_idx) +{ + assert(locals_idx >= 0); + sym.sym->locals_idx = locals_idx; +} + +int +_Py_uop_sym_get_locals_idx(_Py_UopsLocalsPlusSlot sym) +{ + return sym.sym->locals_idx; +} int -_Py_uop_sym_truthiness(_Py_UopsSymbol *sym) +_Py_uop_sym_truthiness(_Py_UopsLocalsPlusSlot sym) { /* There are some non-constant values for * which `bool(val)` always evaluates to @@ -338,7 +362,7 @@ _Py_uop_frame_new( _Py_UOpsContext *ctx, PyCodeObject *co, int curr_stackentries, - _Py_UopsSymbol **args, + _Py_UopsLocalsPlusSlot *args, int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); @@ -363,15 +387,17 @@ _Py_uop_frame_new( } for (int i = arg_len; i < co->co_nlocalsplus; i++) { - _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); - frame->locals[i] = local; + frame->locals[i] = _Py_uop_sym_new_unknown(ctx);; + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + frame->locals[i].sym->locals_idx = i; } // Initialize the stack as well for (int i = 0; i < curr_stackentries; i++) { - _Py_UopsSymbol *stackvar = _Py_uop_sym_new_unknown(ctx); - frame->stack[i] = stackvar; + frame->stack[i] = _Py_uop_sym_new_unknown(ctx); } return frame; @@ -397,7 +423,8 @@ _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) ctx->n_consumed = ctx->locals_and_stack; #ifdef Py_DEBUG // Aids debugging a little. There should never be NULL in the abstract interpreter. for (int i = 0 ; i < MAX_ABSTRACT_INTERP_SIZE; i++) { - ctx->locals_and_stack[i] = NULL; + _Py_UopsLocalsPlusSlot slot = {NULL, 0}; + ctx->locals_and_stack[i] = slot; } #endif @@ -431,10 +458,10 @@ do { \ } \ } while (0) -static _Py_UopsSymbol * +static _Py_UopsLocalsPlusSlot make_bottom(_Py_UOpsContext *ctx) { - _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); + _Py_UopsLocalsPlusSlot sym = _Py_uop_sym_new_unknown(ctx); _Py_uop_sym_set_null(ctx, sym); _Py_uop_sym_set_non_null(ctx, sym); return sym; @@ -450,8 +477,8 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) PyObject *val_43 = NULL; // Use a single 'sym' variable so copy-pasting tests is easier. - _Py_UopsSymbol *sym = _Py_uop_sym_new_unknown(ctx); - if (sym == NULL) { + _Py_UopsLocalsPlusSlot sym = _Py_uop_sym_new_unknown(ctx); + if (sym.sym == NULL) { goto fail; } TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "top is NULL"); @@ -462,7 +489,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(!_Py_uop_sym_is_bottom(sym), "top is bottom"); sym = make_bottom(ctx); - if (sym == NULL) { + if (sym.sym == NULL) { goto fail; } TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "bottom is NULL is not false"); @@ -473,7 +500,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "bottom isn't bottom"); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + if (sym.sym == NULL) { goto fail; } TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "int is NULL"); @@ -498,7 +525,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) assert(_Py_IsImmortal(val_43)); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + if (sym.sym == NULL) { goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); @@ -519,7 +546,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and float) isn't bottom"); sym = _Py_uop_sym_new_type(ctx, &PyLong_Type); - if (sym == NULL) { + if (sym.sym == NULL) { goto fail; } _Py_uop_sym_set_const(ctx, sym, val_42); diff --git a/Python/partial_evaluator_bytecodes.c b/Python/partial_evaluator_bytecodes.c new file mode 100644 index 00000000000000..8ef020e0b7250b --- /dev/null +++ b/Python/partial_evaluator_bytecodes.c @@ -0,0 +1,113 @@ +#include "Python.h" +#include "pycore_optimizer.h" +#include "pycore_uops.h" +#include "pycore_uop_ids.h" +#include "internal/pycore_moduleobject.h" + +#define op(name, ...) /* NAME is ignored */ + +typedef struct _Py_UopsSymbol _Py_UopsSymbol; +typedef struct _Py_UOpsContext _Py_UOpsContext; +typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; + +/* Shortened forms for convenience */ +#define sym_is_not_null _Py_uop_sym_is_not_null +#define sym_is_const _Py_uop_sym_is_const +#define sym_get_const _Py_uop_sym_get_const +#define sym_new_unknown _Py_uop_sym_new_unknown +#define sym_new_not_null _Py_uop_sym_new_not_null +#define sym_new_type _Py_uop_sym_new_type +#define sym_is_null _Py_uop_sym_is_null +#define sym_new_const _Py_uop_sym_new_const +#define sym_new_null _Py_uop_sym_new_null +#define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version +#define sym_get_type _Py_uop_sym_get_type +#define sym_has_type _Py_uop_sym_has_type +#define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) +#define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) +#define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) +#define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) +#define sym_is_bottom _Py_uop_sym_is_bottom +#define frame_new _Py_uop_frame_new +#define frame_pop _Py_uop_frame_pop + +extern int +optimize_to_bool( + _PyUOpInstruction *this_instr, + _Py_UOpsContext *ctx, + _Py_UopsSymbol *value, + _Py_UopsSymbol **result_ptr); + +extern void +eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit); + +extern PyCodeObject *get_code(_PyUOpInstruction *op); + +static int +dummy_func(void) { + +// BEGIN BYTECODES // + + override op(_LOAD_FAST, (-- value)) { + value = GETLOCAL(oparg); + sym_set_locals_idx(value, oparg); + SET_STATIC_INST(); + value.is_virtual = true; + } + + override op(_LOAD_FAST_AND_CLEAR, (-- value)) { + value = GETLOCAL(oparg); + GETLOCAL(oparg) = sym_new_null(ctx); + } + + override op(_LOAD_CONST, (-- value)) { + // Should've all been converted by specializer. + Py_UNREACHABLE(); + } + + override op(_LOAD_CONST_INLINE, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); + SET_STATIC_INST(); + value.is_virtual = true; + } + + override op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { + value = sym_new_const(ctx, ptr); + SET_STATIC_INST(); + value.is_virtual = true; + } + + override op(_STORE_FAST, (value --)) { + // Gets rid of stores by the same load + if (value.is_virtual && oparg == sym_get_locals_idx(value)) { + SET_STATIC_INST(); + } + else { + reify_shadow_stack(ctx); + value.is_virtual = false; + } + GETLOCAL(oparg) = value; + } + + override op(_POP_TOP, (pop --)) { + if (pop.is_virtual) { + SET_STATIC_INST(); + } + else { + reify_shadow_stack(ctx); + } + } + + override op(_NOP, (--)) { + SET_STATIC_INST(); + } + + override op(_CHECK_STACK_SPACE_OPERAND, ( -- )) { + (void)framesize; + } + +// END BYTECODES // + +} diff --git a/Python/partial_evaluator_cases.c.h b/Python/partial_evaluator_cases.c.h new file mode 100644 index 00000000000000..6781a9f3a13373 --- /dev/null +++ b/Python/partial_evaluator_cases.c.h @@ -0,0 +1,2423 @@ +// This file is generated by Tools/cases_generator/optimizer_generator.py +// from: +// Python/optimizer_bytecodes.c, Python/partial_evaluator_bytecodes.c +// Do not edit! + + case _NOP: { + SET_STATIC_INST(); + break; + } + + case _CHECK_PERIODIC: { + break; + } + + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { + break; + } + + /* _QUICKEN_RESUME is not a viable micro-op for tier 2 */ + + case _RESUME_CHECK: { + break; + } + + /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ + + case _LOAD_FAST_CHECK: { + _Py_UopsLocalsPlusSlot value; + value = GETLOCAL(oparg); + // We guarantee this will error - just bail and don't optimize it. + if (sym_is_null(value)) { + ctx->done = true; + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST: { + _Py_UopsLocalsPlusSlot value; + value = GETLOCAL(oparg); + sym_set_locals_idx(value, oparg); + SET_STATIC_INST(); + value.is_virtual = true; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_FAST_AND_CLEAR: { + _Py_UopsLocalsPlusSlot value; + value = GETLOCAL(oparg); + GETLOCAL(oparg) = sym_new_null(ctx); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST: { + _Py_UopsLocalsPlusSlot value; + // Should've all been converted by specializer. + Py_UNREACHABLE(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_FAST: { + _Py_UopsLocalsPlusSlot value; + value = stack_pointer[-1]; + // Gets rid of stores by the same load + if (value.is_virtual && oparg == sym_get_locals_idx(value)) { + SET_STATIC_INST(); + } + else { + reify_shadow_stack(ctx); + value.is_virtual = false; + } + GETLOCAL(oparg) = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _POP_TOP: { + _Py_UopsLocalsPlusSlot pop; + pop = stack_pointer[-1]; + if (pop.is_virtual) { + SET_STATIC_INST(); + } + else { + reify_shadow_stack(ctx); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_NULL: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _END_SEND: { + _Py_UopsLocalsPlusSlot value; + value = sym_new_not_null(ctx); + stack_pointer[-2] = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNARY_NEGATIVE: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _UNARY_NOT: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL: { + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + } + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_BOOL: { + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyBool_Type); + res = value; + } + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_INT: { + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyLong_Type); + res = sym_new_type(ctx, &PyBool_Type); + } + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_LIST: { + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_type(value, &PyList_Type); + res = sym_new_type(ctx, &PyBool_Type); + } + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_NONE: { + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + sym_set_const(value, Py_None); + res = sym_new_const(ctx, Py_False); + } + stack_pointer[-1] = res; + break; + } + + case _TO_BOOL_STR: { + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot res; + value = stack_pointer[-1]; + if (!optimize_to_bool(this_instr, ctx, value, &res)) { + res = sym_new_type(ctx, &PyBool_Type); + sym_set_type(value, &PyUnicode_Type); + } + stack_pointer[-1] = res; + break; + } + + case _REPLACE_WITH_TRUE: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _UNARY_INVERT: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _GUARD_BOTH_INT: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyLong_Type)) { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_INT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyLong_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_INT, 0, 0); + } + } + sym_set_type(left, &PyLong_Type); + sym_set_type(right, &PyLong_Type); + break; + } + + case _GUARD_NOS_INT: { + break; + } + + case _GUARD_TOS_INT: { + break; + } + + case _BINARY_OP_MULTIPLY_INT: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_INT: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_INT: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyLong_Type) && sym_matches_type(right, &PyLong_Type)) + { + assert(PyLong_CheckExact(sym_get_const(left))); + assert(PyLong_CheckExact(sym_get_const(right))); + PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(left), + (PyLongObject *)sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and add tests! + } + else { + res = sym_new_type(ctx, &PyLong_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_BOTH_FLOAT: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyFloat_Type)) { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + REPLACE_OP(this_instr, _GUARD_TOS_FLOAT, 0, 0); + } + } + else { + if (sym_matches_type(right, &PyFloat_Type)) { + REPLACE_OP(this_instr, _GUARD_NOS_FLOAT, 0, 0); + } + } + sym_set_type(left, &PyFloat_Type); + sym_set_type(right, &PyFloat_Type); + break; + } + + case _GUARD_NOS_FLOAT: { + break; + } + + case _GUARD_TOS_FLOAT: { + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) * + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_ADD_FLOAT: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) + + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyFloat_Type) && sym_matches_type(right, &PyFloat_Type)) + { + assert(PyFloat_CheckExact(sym_get_const(left))); + assert(PyFloat_CheckExact(sym_get_const(right))); + PyObject *temp = PyFloat_FromDouble( + PyFloat_AS_DOUBLE(sym_get_const(left)) - + PyFloat_AS_DOUBLE(sym_get_const(right))); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + // TODO gh-115506: + // replace opcode with constant propagated one and update tests! + } + else { + res = sym_new_type(ctx, &PyFloat_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_BOTH_UNICODE: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_matches_type(left, &PyUnicode_Type) && + sym_matches_type(right, &PyUnicode_Type)) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + sym_set_type(left, &PyUnicode_Type); + sym_set_type(left, &PyUnicode_Type); + break; + } + + case _BINARY_OP_ADD_UNICODE: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (sym_is_const(left) && sym_is_const(right) && + sym_matches_type(left, &PyUnicode_Type) && sym_matches_type(right, &PyUnicode_Type)) { + PyObject *temp = PyUnicode_Concat(sym_get_const(left), sym_get_const(right)); + if (temp == NULL) { + goto error; + } + res = sym_new_const(ctx, temp); + Py_DECREF(temp); + } + else { + res = sym_new_type(ctx, &PyUnicode_Type); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP_INPLACE_ADD_UNICODE: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SLICE: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SLICE: { + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_LIST_INT: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_STR_INT: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_TUPLE_INT: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_DICT: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_SUBSCR_CHECK_FUNC: { + break; + } + + case _BINARY_SUBSCR_INIT_CALL: { + _Py_UopsLocalsPlusSlot sub; + _Py_UopsLocalsPlusSlot container; + _Py_UopsLocalsPlusSlot new_frame; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + (void)container; + (void)sub; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + ctx->done = true; + stack_pointer[-2] = new_frame; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LIST_APPEND: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SET_ADD: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR_LIST_INT: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_SUBSCR_DICT: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_SUBSCR: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_INTRINSIC_1: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _CALL_INTRINSIC_2: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _RETURN_VALUE: { + _Py_UopsLocalsPlusSlot retval; + _Py_UopsLocalsPlusSlot res; + retval = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = retval; + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GET_AITER: { + _Py_UopsLocalsPlusSlot iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + case _GET_ANEXT: { + _Py_UopsLocalsPlusSlot awaitable; + awaitable = sym_new_not_null(ctx); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GET_AWAITABLE: { + _Py_UopsLocalsPlusSlot iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + /* _SEND is not a viable micro-op for tier 2 */ + + case _SEND_GEN_FRAME: { + // We are about to hit the end of the trace: + ctx->done = true; + break; + } + + case _YIELD_VALUE: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_unknown(ctx); + stack_pointer[-1] = res; + break; + } + + case _POP_EXCEPT: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_COMMON_CONSTANT: { + _Py_UopsLocalsPlusSlot value; + value = sym_new_not_null(ctx); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_BUILD_CLASS: { + _Py_UopsLocalsPlusSlot bc; + bc = sym_new_not_null(ctx); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_NAME: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_NAME: { + break; + } + + case _UNPACK_SEQUENCE: { + _Py_UopsLocalsPlusSlot seq; + _Py_UopsLocalsPlusSlot *values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + /* This has to be done manually */ + (void)seq; + for (int i = 0; i < oparg; i++) { + values[i] = sym_new_unknown(ctx); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_SEQUENCE_TWO_TUPLE: { + _Py_UopsLocalsPlusSlot val1; + _Py_UopsLocalsPlusSlot val0; + val1 = sym_new_not_null(ctx); + val0 = sym_new_not_null(ctx); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_SEQUENCE_TUPLE: { + _Py_UopsLocalsPlusSlot *values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_not_null(ctx); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_SEQUENCE_LIST: { + _Py_UopsLocalsPlusSlot *values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_not_null(ctx); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _UNPACK_EX: { + _Py_UopsLocalsPlusSlot seq; + _Py_UopsLocalsPlusSlot *values; + seq = stack_pointer[-1]; + values = &stack_pointer[-1]; + /* This has to be done manually */ + (void)seq; + int totalargs = (oparg & 0xFF) + (oparg >> 8) + 1; + for (int i = 0; i < totalargs; i++) { + values[i] = sym_new_unknown(ctx); + } + stack_pointer += (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_ATTR: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_ATTR: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_GLOBAL: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_GLOBAL: { + break; + } + + case _LOAD_LOCALS: { + _Py_UopsLocalsPlusSlot locals; + locals = sym_new_not_null(ctx); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _LOAD_FROM_DICT_OR_GLOBALS is not a viable micro-op for tier 2 */ + + case _LOAD_NAME: { + _Py_UopsLocalsPlusSlot v; + v = sym_new_not_null(ctx); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_GLOBAL: { + _Py_UopsLocalsPlusSlot *res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + res = &stack_pointer[0]; + for (int _i = 1; --_i >= 0;) { + res[_i] = sym_new_not_null(ctx); + } + null = sym_new_null(ctx); + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_GLOBALS_VERSION: { + break; + } + + case _GUARD_BUILTINS_VERSION: { + break; + } + + case _LOAD_GLOBAL_MODULE: { + _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_GLOBAL_BUILTINS: { + _Py_UopsLocalsPlusSlot res; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + res = sym_new_not_null(ctx); + null = sym_new_null(ctx); + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DELETE_FAST: { + break; + } + + case _MAKE_CELL: { + break; + } + + case _DELETE_DEREF: { + break; + } + + case _LOAD_FROM_DICT_OR_DEREF: { + _Py_UopsLocalsPlusSlot value; + value = sym_new_not_null(ctx); + stack_pointer[-1] = value; + break; + } + + case _LOAD_DEREF: { + _Py_UopsLocalsPlusSlot value; + value = sym_new_not_null(ctx); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_DEREF: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COPY_FREE_VARS: { + break; + } + + case _BUILD_STRING: { + _Py_UopsLocalsPlusSlot str; + str = sym_new_not_null(ctx); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_TUPLE: { + _Py_UopsLocalsPlusSlot tup; + tup = sym_new_not_null(ctx); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_LIST: { + _Py_UopsLocalsPlusSlot list; + list = sym_new_not_null(ctx); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LIST_EXTEND: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SET_UPDATE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_SET: { + _Py_UopsLocalsPlusSlot set; + set = sym_new_not_null(ctx); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_MAP: { + _Py_UopsLocalsPlusSlot map; + map = sym_new_not_null(ctx); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SETUP_ANNOTATIONS: { + break; + } + + case _DICT_UPDATE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DICT_MERGE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MAP_ADD: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 */ + + case _LOAD_SUPER_ATTR_ATTR: { + _Py_UopsLocalsPlusSlot attr_st; + attr_st = sym_new_not_null(ctx); + stack_pointer[-3] = attr_st; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_SUPER_ATTR_METHOD: { + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_not_null(ctx); + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + owner = stack_pointer[-1]; + (void)owner; + attr = sym_new_not_null(ctx); + if (oparg & 1) { + self_or_null = sym_new_unknown(ctx); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_TYPE_VERSION: { + _Py_UopsLocalsPlusSlot owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)this_instr->operand; + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + } + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + owner = stack_pointer[-1]; + uint16_t offset = (uint16_t)this_instr->operand; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)offset; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_ATTR_MODULE: { + _Py_UopsLocalsPlusSlot owner; + owner = stack_pointer[-1]; + uint32_t dict_version = (uint32_t)this_instr->operand; + (void)dict_version; + if (sym_is_const(owner)) { + PyObject *cnst = sym_get_const(owner); + if (PyModule_CheckExact(cnst)) { + PyModuleObject *mod = (PyModuleObject *)cnst; + PyObject *dict = mod->md_dict; + uint64_t watched_mutations = get_mutations(dict); + if (watched_mutations < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) { + PyDict_Watch(GLOBALS_WATCHER_ID, dict); + _Py_BloomFilter_Add(dependencies, dict); + this_instr->opcode = _NOP; + } + } + } + break; + } + + case _LOAD_ATTR_MODULE: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)this_instr->operand; + (void)index; + null = sym_new_null(ctx); + attr = (_Py_UopsLocalsPlusSlot){NULL, 0}; + if (this_instr[-1].opcode == _NOP) { + // Preceding _CHECK_ATTR_MODULE was removed: mod is const and dict is watched. + assert(sym_is_const(owner)); + PyModuleObject *mod = (PyModuleObject *)sym_get_const(owner); + assert(PyModule_CheckExact(mod)); + PyObject *dict = mod->md_dict; + PyObject *res = convert_global_to_const(this_instr, dict); + if (res != NULL) { + this_instr[-1].opcode = _POP_TOP; + attr = sym_new_const(ctx, res); + } + } + if (attr.sym == NULL) { + /* No conversion made. We don't know what `attr` is. */ + attr = sym_new_not_null(ctx); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_ATTR_WITH_HINT: { + break; + } + + case _LOAD_ATTR_WITH_HINT: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + owner = stack_pointer[-1]; + uint16_t hint = (uint16_t)this_instr->operand; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)hint; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_SLOT: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)this_instr->operand; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)index; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_ATTR_CLASS: { + break; + } + + case _LOAD_ATTR_CLASS: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot null = (_Py_UopsLocalsPlusSlot){NULL, 0}; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + attr = sym_new_not_null(ctx); + null = sym_new_null(ctx); + (void)descr; + (void)owner; + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_PROPERTY_FRAME: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot new_frame; + owner = stack_pointer[-1]; + PyObject *fget = (PyObject *)this_instr->operand; + (void)fget; + (void)owner; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + ctx->done = true; + stack_pointer[-1] = new_frame; + break; + } + + /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */ + + case _GUARD_DORV_NO_DICT: { + break; + } + + case _STORE_ATTR_INSTANCE_VALUE: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_ATTR_WITH_HINT: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _STORE_ATTR_SLOT: { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + if (oparg & 16) { + res = sym_new_type(ctx, &PyBool_Type); + } + else { + res = _Py_uop_sym_new_not_null(ctx); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP_FLOAT: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP_INT: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COMPARE_OP_STR: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IS_OP: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + (void)left; + (void)right; + res = sym_new_type(ctx, &PyBool_Type); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_SET: { + _Py_UopsLocalsPlusSlot b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONTAINS_OP_DICT: { + _Py_UopsLocalsPlusSlot b; + b = sym_new_not_null(ctx); + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_EG_MATCH: { + _Py_UopsLocalsPlusSlot rest; + _Py_UopsLocalsPlusSlot match; + rest = sym_new_not_null(ctx); + match = sym_new_not_null(ctx); + stack_pointer[-2] = rest; + stack_pointer[-1] = match; + break; + } + + case _CHECK_EXC_MATCH: { + _Py_UopsLocalsPlusSlot b; + b = sym_new_not_null(ctx); + stack_pointer[-1] = b; + break; + } + + case _IMPORT_NAME: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _IMPORT_FROM: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + case _IS_NONE: { + _Py_UopsLocalsPlusSlot b; + b = sym_new_not_null(ctx); + stack_pointer[-1] = b; + break; + } + + case _GET_LEN: { + _Py_UopsLocalsPlusSlot len; + len = sym_new_not_null(ctx); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_CLASS: { + _Py_UopsLocalsPlusSlot attrs; + attrs = sym_new_not_null(ctx); + stack_pointer[-3] = attrs; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_MAPPING: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_SEQUENCE: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MATCH_KEYS: { + _Py_UopsLocalsPlusSlot values_or_none; + values_or_none = sym_new_not_null(ctx); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GET_ITER: { + _Py_UopsLocalsPlusSlot iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + case _GET_YIELD_FROM_ITER: { + _Py_UopsLocalsPlusSlot iter; + iter = sym_new_not_null(ctx); + stack_pointer[-1] = iter; + break; + } + + /* _FOR_ITER is not a viable micro-op for tier 2 */ + + case _FOR_ITER_TIER_TWO: { + _Py_UopsLocalsPlusSlot next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_FOR_ITER is not a viable micro-op for tier 2 */ + + case _ITER_CHECK_LIST: { + break; + } + + /* _ITER_JUMP_LIST is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_LIST: { + break; + } + + case _ITER_NEXT_LIST: { + _Py_UopsLocalsPlusSlot next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _ITER_CHECK_TUPLE: { + break; + } + + /* _ITER_JUMP_TUPLE is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_TUPLE: { + break; + } + + case _ITER_NEXT_TUPLE: { + _Py_UopsLocalsPlusSlot next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _ITER_CHECK_RANGE: { + break; + } + + /* _ITER_JUMP_RANGE is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_EXHAUSTED_RANGE: { + break; + } + + case _ITER_NEXT_RANGE: { + _Py_UopsLocalsPlusSlot iter; + _Py_UopsLocalsPlusSlot next; + iter = stack_pointer[-1]; + next = sym_new_type(ctx, &PyLong_Type); + (void)iter; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _FOR_ITER_GEN_FRAME: { + /* We are about to hit the end of the trace */ + ctx->done = true; + break; + } + + case _LOAD_SPECIAL: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self_or_null; + owner = stack_pointer[-1]; + (void)owner; + attr = sym_new_not_null(ctx); + self_or_null = sym_new_unknown(ctx); + stack_pointer[-1] = attr; + stack_pointer[0] = self_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _WITH_EXCEPT_START: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_EXC_INFO: { + _Py_UopsLocalsPlusSlot prev_exc; + _Py_UopsLocalsPlusSlot new_exc; + prev_exc = sym_new_not_null(ctx); + new_exc = sym_new_not_null(ctx); + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: { + break; + } + + case _GUARD_KEYS_VERSION: { + break; + } + + case _LOAD_ATTR_METHOD_WITH_VALUES: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + attr = sym_new_not_null(ctx); + self = owner; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_METHOD_NO_DICT: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + attr = sym_new_not_null(ctx); + self = owner; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { + _Py_UopsLocalsPlusSlot attr; + attr = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + break; + } + + case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { + _Py_UopsLocalsPlusSlot attr; + attr = sym_new_not_null(ctx); + stack_pointer[-1] = attr; + break; + } + + case _CHECK_ATTR_METHOD_LAZY_DICT: { + break; + } + + case _LOAD_ATTR_METHOD_LAZY_DICT: { + _Py_UopsLocalsPlusSlot owner; + _Py_UopsLocalsPlusSlot attr; + _Py_UopsLocalsPlusSlot self = (_Py_UopsLocalsPlusSlot){NULL, 0}; + owner = stack_pointer[-1]; + PyObject *descr = (PyObject *)this_instr->operand; + (void)descr; + attr = sym_new_not_null(ctx); + self = owner; + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _MAYBE_EXPAND_METHOD: { + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot func; + _Py_UopsLocalsPlusSlot maybe_self; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + (void)callable; + (void)self_or_null; + (void)args; + func = sym_new_not_null(ctx); + maybe_self = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + break; + } + + /* _DO_CALL is not a viable micro-op for tier 2 */ + + /* _MONITOR_CALL is not a viable micro-op for tier 2 */ + + case _PY_FRAME_GENERAL: { + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ + (void)callable; + (void)self_or_null; + (void)args; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + ctx->done = true; + stack_pointer[-2 - oparg] = new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION: { + break; + } + + case _CHECK_METHOD_VERSION: { + break; + } + + case _EXPAND_METHOD: { + _Py_UopsLocalsPlusSlot method; + _Py_UopsLocalsPlusSlot *self; + self = &stack_pointer[-1 - oparg]; + method = sym_new_not_null(ctx); + for (int _i = 1; --_i >= 0;) { + self[_i] = sym_new_not_null(ctx); + } + stack_pointer[-2 - oparg] = method; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE: { + break; + } + + case _CALL_NON_PY_GENERAL: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsLocalsPlusSlot null; + _Py_UopsLocalsPlusSlot callable; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + sym_set_null(null); + sym_set_type(callable, &PyMethod_Type); + break; + } + + case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot func; + _Py_UopsLocalsPlusSlot self; + callable = stack_pointer[-2 - oparg]; + (void)callable; + func = sym_new_not_null(ctx); + self = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_PEP_523: { + /* Setting the eval frame function invalidates + * all executors, so no need to check dynamically */ + if (_PyInterpreterState_GET()->eval_frame == NULL) { + REPLACE_OP(this_instr, _NOP, 0 ,0); + } + break; + } + + case _CHECK_FUNCTION_EXACT_ARGS: { + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + sym_set_type(callable, &PyFunction_Type); + (void)self_or_null; + break; + } + + case _CHECK_STACK_SPACE: { + assert(corresponding_check_stack == NULL); + corresponding_check_stack = this_instr; + break; + } + + case _INIT_CALL_PY_EXACT_ARGS: { + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + (void)callable; + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + uint64_t push_operand = (this_instr + 2)->operand; + if (push_operand & 1) { + co = (PyCodeObject *)(push_operand & ~1); + DPRINTF(3, "code=%p ", co); + assert(PyCode_Check(co)); + } + else { + PyFunctionObject *func = (PyFunctionObject *)push_operand; + DPRINTF(3, "func=%p ", func); + if (func == NULL) { + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + ctx->done = true; + break; + } + co = (PyCodeObject *)func->func_code; + DPRINTF(3, "code=%p ", co); + } + assert(self_or_null.sym != NULL); + assert(args != NULL); + if (sym_is_not_null(self_or_null)) { + // Bound method fiddling, same as _INIT_CALL_PY_EXACT_ARGS in VM + args--; + argcount++; + } + if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, args, argcount); + } else { + new_frame.sym = (_Py_UopsSymbol *)frame_new(ctx, co, 0, NULL, 0); + } + stack_pointer[-2 - oparg] = new_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _PUSH_FRAME: { + _Py_UopsLocalsPlusSlot new_frame; + new_frame = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = (_Py_UOpsAbstractFrame *)new_frame.sym; + ctx->curr_frame_depth++; + stack_pointer = ((_Py_UOpsAbstractFrame *)new_frame.sym)->stack_pointer; + co = get_code(this_instr); + if (co == NULL) { + // should be about to _EXIT_TRACE anyway + ctx->done = true; + break; + } + /* Stack space handling */ + int framesize = co->co_framesize; + assert(framesize > 0); + curr_space += framesize; + if (curr_space < 0 || curr_space > INT32_MAX) { + // won't fit in signed 32-bit int + ctx->done = true; + break; + } + max_space = curr_space > max_space ? curr_space : max_space; + if (first_valid_check_stack == NULL) { + first_valid_check_stack = corresponding_check_stack; + } + else if (corresponding_check_stack) { + // delete all but the first valid _CHECK_STACK_SPACE + corresponding_check_stack->opcode = _NOP; + } + corresponding_check_stack = NULL; + break; + } + + case _CALL_TYPE_1: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_STR_1: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_TUPLE_1: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_AND_ALLOCATE_OBJECT: { + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot init; + args = &stack_pointer[-oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-oparg]; + uint32_t type_version = (uint32_t)this_instr->operand; + (void)type_version; + (void)callable; + (void)null; + (void)args; + self = sym_new_not_null(ctx); + init = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = self; + stack_pointer[-1 - oparg] = init; + break; + } + + case _CREATE_INIT_FRAME: { + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot init; + _Py_UopsLocalsPlusSlot self; + _Py_UopsLocalsPlusSlot init_frame; + args = &stack_pointer[-oparg]; + init = stack_pointer[-1 - oparg]; + self = stack_pointer[-2 - oparg]; + (void)self; + (void)init; + (void)args; + init_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + ctx->done = true; + stack_pointer[-2 - oparg] = init_frame; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _EXIT_INIT_CHECK: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_CLASS: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_O: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_FAST: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_LEN: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_ISINSTANCE: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_LIST_APPEND: { + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_O: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_NOARGS: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CALL_METHOD_DESCRIPTOR_FAST: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_CALL_KW is not a viable micro-op for tier 2 */ + + /* _DO_CALL_KW is not a viable micro-op for tier 2 */ + + case _PY_FRAME_KW: { + _Py_UopsLocalsPlusSlot kwnames; + _Py_UopsLocalsPlusSlot *args; + _Py_UopsLocalsPlusSlot self_or_null; + _Py_UopsLocalsPlusSlot callable; + _Py_UopsLocalsPlusSlot new_frame; + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + (void)callable; + (void)self_or_null; + (void)args; + (void)kwnames; + new_frame = (_Py_UopsLocalsPlusSlot){NULL, 0}; + ctx->done = true; + stack_pointer[-3 - oparg] = new_frame; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION_VERSION_KW: { + break; + } + + case _CHECK_METHOD_VERSION_KW: { + break; + } + + case _EXPAND_METHOD_KW: { + _Py_UopsLocalsPlusSlot method; + _Py_UopsLocalsPlusSlot *self; + _Py_UopsLocalsPlusSlot kwnames; + self = &stack_pointer[-2 - oparg]; + method = sym_new_not_null(ctx); + for (int _i = 1; --_i >= 0;) { + self[_i] = sym_new_not_null(ctx); + } + kwnames = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = method; + stack_pointer[-1] = kwnames; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE_KW: { + break; + } + + case _CALL_KW_NON_PY: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + /* _INSTRUMENTED_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + /* __DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */ + + case _MAKE_FUNCTION: { + _Py_UopsLocalsPlusSlot func; + func = sym_new_not_null(ctx); + stack_pointer[-1] = func; + break; + } + + case _SET_FUNCTION_ATTRIBUTE: { + _Py_UopsLocalsPlusSlot func_st; + func_st = sym_new_not_null(ctx); + stack_pointer[-2] = func_st; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _RETURN_GENERATOR: { + _Py_UopsLocalsPlusSlot res; + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); + stack_pointer = ctx->frame->stack_pointer; + res = sym_new_unknown(ctx); + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; + co = get_code(this_instr); + if (co == NULL) { + // might be impossible, but bailing is still safe + ctx->done = true; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BUILD_SLICE: { + _Py_UopsLocalsPlusSlot slice; + slice = sym_new_not_null(ctx); + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CONVERT_VALUE: { + _Py_UopsLocalsPlusSlot result; + result = sym_new_not_null(ctx); + stack_pointer[-1] = result; + break; + } + + case _FORMAT_SIMPLE: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + + case _FORMAT_WITH_SPEC: { + _Py_UopsLocalsPlusSlot res; + res = sym_new_not_null(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _COPY: { + _Py_UopsLocalsPlusSlot bottom; + _Py_UopsLocalsPlusSlot top; + bottom = stack_pointer[-1 - (oparg-1)]; + assert(oparg > 0); + top = bottom; + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _BINARY_OP: { + _Py_UopsLocalsPlusSlot right; + _Py_UopsLocalsPlusSlot left; + _Py_UopsLocalsPlusSlot res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyTypeObject *ltype = sym_get_type(left); + PyTypeObject *rtype = sym_get_type(right); + if (ltype != NULL && (ltype == &PyLong_Type || ltype == &PyFloat_Type) && + rtype != NULL && (rtype == &PyLong_Type || rtype == &PyFloat_Type)) + { + if (oparg != NB_TRUE_DIVIDE && oparg != NB_INPLACE_TRUE_DIVIDE && + ltype == &PyLong_Type && rtype == &PyLong_Type) { + /* If both inputs are ints and the op is not division the result is an int */ + res = sym_new_type(ctx, &PyLong_Type); + } + else { + /* For any other op combining ints/floats the result is a float */ + res = sym_new_type(ctx, &PyFloat_Type); + } + } + res = sym_new_unknown(ctx); + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _SWAP: { + _Py_UopsLocalsPlusSlot top; + _Py_UopsLocalsPlusSlot bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + stack_pointer[-2 - (oparg-2)] = top; + stack_pointer[-1] = bottom; + break; + } + + /* _INSTRUMENTED_LINE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_INSTRUCTION is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_JUMP_FORWARD is not a viable micro-op for tier 2 */ + + /* _MONITOR_JUMP_BACKWARD is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NONE is not a viable micro-op for tier 2 */ + + /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ + + case _GUARD_IS_TRUE_POP: { + _Py_UopsLocalsPlusSlot flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_True); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_IS_FALSE_POP: { + _Py_UopsLocalsPlusSlot flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_False); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_IS_NONE_POP: { + _Py_UopsLocalsPlusSlot flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, !Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, true); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _GUARD_IS_NOT_NONE_POP: { + _Py_UopsLocalsPlusSlot flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, false); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _JUMP_TO_TOP: { + ctx->done = true; + break; + } + + case _SET_IP: { + break; + } + + case _CHECK_STACK_SPACE_OPERAND: { + uint32_t framesize = (uint32_t)this_instr->operand; + (void)framesize; + break; + } + + case _SAVE_RETURN_OFFSET: { + break; + } + + case _EXIT_TRACE: { + PyObject *exit_p = (PyObject *)this_instr->operand; + (void)exit_p; + ctx->done = true; + break; + } + + case _CHECK_VALIDITY: { + break; + } + + case _LOAD_CONST_INLINE: { + _Py_UopsLocalsPlusSlot value; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + SET_STATIC_INST(); + value.is_virtual = true; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST_INLINE_BORROW: { + _Py_UopsLocalsPlusSlot value; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + SET_STATIC_INST(); + value.is_virtual = true; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _POP_TOP_LOAD_CONST_INLINE_BORROW: { + _Py_UopsLocalsPlusSlot value; + value = sym_new_not_null(ctx); + stack_pointer[-1] = value; + break; + } + + case _LOAD_CONST_INLINE_WITH_NULL: { + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot null; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _LOAD_CONST_INLINE_BORROW_WITH_NULL: { + _Py_UopsLocalsPlusSlot value; + _Py_UopsLocalsPlusSlot null; + PyObject *ptr = (PyObject *)this_instr->operand; + value = sym_new_const(ctx, ptr); + null = sym_new_null(ctx); + stack_pointer[0] = value; + stack_pointer[1] = null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _CHECK_FUNCTION: { + break; + } + + case _INTERNAL_INCREMENT_OPT_COUNTER: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _DYNAMIC_EXIT: { + break; + } + + case _START_EXECUTOR: { + break; + } + + case _FATAL_ERROR: { + break; + } + + case _CHECK_VALIDITY_AND_SET_IP: { + break; + } + + case _DEOPT: { + break; + } + + case _ERROR_POP_N: { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + break; + } + + case _TIER2_RESUME_CHECK: { + break; + } + diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 3a73f65f8ff7b3..6c8250f67073b4 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -84,6 +84,7 @@ def clean_lines(text): Python/generated_cases.c.h Python/executor_cases.c.h Python/optimizer_cases.c.h +Python/partial_evaluator_cases.c.h # not actually source Python/bytecodes.c diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index a4ce207703edcd..1d17c80ed0a67c 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -23,6 +23,7 @@ class Properties: has_free: bool side_exit: bool pure: bool + static: bool = False tier: int | None = None oparg_and_1: bool = False const_oparg: int = -1 @@ -688,6 +689,7 @@ def compute_properties(op: parser.InstDef) -> Properties: and not has_free, has_free=has_free, pure="pure" in op.annotations, + static="_static" in op.annotations, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 4cfd4ad3d05988..682b7be6e50de1 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -266,6 +266,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.oparg_and_1: flags.append("HAS_OPARG_AND_1_FLAG") + if p.static: + flags.append("HAS_STATIC_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index d5831593215f76..c171e0b94da5ed 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -226,6 +226,7 @@ def choice(*opts: str) -> str: "replicate", "tier1", "tier2", + "_static", } __all__ = [] diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 2ad7604af9cc0d..176596719453cb 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -52,6 +52,7 @@ "PASSTHROUGH", "OPARG_AND_1", "ERROR_NO_POP", + "STATIC", ] diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index b74f627235ad84..8e3e6e2d6c6022 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -35,10 +35,10 @@ def validate_uop(override: Uop, uop: Uop) -> None: def type_name(var: StackItem) -> str: if var.is_array(): - return f"_Py_UopsSymbol **" + return f"_Py_UopsLocalsPlusSlot *" if var.type: return var.type - return f"_Py_UopsSymbol *" + return f"_Py_UopsLocalsPlusSlot " def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: @@ -48,7 +48,7 @@ def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: if var.name not in variables: variables.add(var.name) if var.condition: - out.emit(f"{type_name(var)}{var.name} = NULL;\n") + out.emit(f"{type_name(var)}{var.name} = (_Py_UopsLocalsPlusSlot){{NULL, 0}};\n") else: out.emit(f"{type_name(var)}{var.name};\n") for var in uop.stack.outputs: @@ -57,7 +57,7 @@ def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None: if var.name not in variables: variables.add(var.name) if var.condition: - out.emit(f"{type_name(var)}{var.name} = NULL;\n") + out.emit(f"{type_name(var)}{var.name} = (_Py_UopsLocalsPlusSlot){{NULL, 0}};\n") else: out.emit(f"{type_name(var)}{var.name};\n") @@ -141,7 +141,7 @@ def write_uop( local = Local.local(var) stack.push(local) out.start_line() - stack.flush(out, cast_type="_Py_UopsSymbol *", extract_bits=True) + stack.flush(out, cast_type="", extract_bits=True) except StackError as ex: raise analysis_error(ex.args[0], uop.body[0]) diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index de4d900563ee0b..f7623a2c60fe64 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -256,7 +256,7 @@ def _do_emit( cast_type: str = "uintptr_t", extract_bits: bool = False, ) -> None: - cast = f"({cast_type})" if var.type else "" + cast = f"({cast_type})" if var.type and cast_type else "" bits = ".bits" if cast and not extract_bits else "" if var.condition == "0": return