From ed3d91febcd46c58f46688d4b62e1b04f15b0c53 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 8 Apr 2024 10:23:01 +0100 Subject: [PATCH 01/15] Make insturctionSequence a PyObject. Simplify tests. --- .../pycore_global_objects_fini_generated.h | 4 + Include/internal/pycore_global_strings.h | 4 + .../internal/pycore_instruction_sequence.h | 13 +- .../internal/pycore_runtime_init_generated.h | 4 + .../internal/pycore_unicodeobject_generated.h | 12 + Lib/test/support/bytecode_helper.py | 37 ++- Lib/test/test_compile.py | 42 +++ Modules/_testinternalcapi.c | 15 + Modules/clinic/_testinternalcapi.c.h | 20 +- Objects/object.c | 2 + Python/clinic/instruction_sequence.c.h | 304 ++++++++++++++++++ Python/compile.c | 69 +--- Python/instruction_sequence.c | 287 +++++++++++++++++ 13 files changed, 752 insertions(+), 61 deletions(-) create mode 100644 Python/clinic/instruction_sequence.c.h diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9aa34f5927dea8..28980d91762aeb 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -863,6 +863,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_stacksize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_varnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(col_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); @@ -912,6 +913,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_col_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_lineno)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(endpos)); @@ -1032,6 +1034,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwdefaults)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(label)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_exc)); @@ -1095,6 +1098,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(narg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_limit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(newline)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 9a0d42f6f12a1e..c369e66dcb5d18 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -352,6 +352,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(co_stacksize) STRUCT_FOR_ID(co_varnames) STRUCT_FOR_ID(code) + STRUCT_FOR_ID(col_offset) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) STRUCT_FOR_ID(compile_mode) @@ -401,6 +402,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(encode) STRUCT_FOR_ID(encoding) STRUCT_FOR_ID(end) + STRUCT_FOR_ID(end_col_offset) STRUCT_FOR_ID(end_lineno) STRUCT_FOR_ID(end_offset) STRUCT_FOR_ID(endpos) @@ -521,6 +523,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(kw1) STRUCT_FOR_ID(kw2) STRUCT_FOR_ID(kwdefaults) + STRUCT_FOR_ID(label) STRUCT_FOR_ID(lambda) STRUCT_FOR_ID(last) STRUCT_FOR_ID(last_exc) @@ -584,6 +587,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(namespaces) STRUCT_FOR_ID(narg) STRUCT_FOR_ID(ndigits) + STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) STRUCT_FOR_ID(new_limit) STRUCT_FOR_ID(newline) diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index b57484fa05309f..e6d25bbdfad3c7 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -5,10 +5,13 @@ # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_symtable.h" + #ifdef __cplusplus extern "C" { #endif + typedef struct { int h_label; int h_startdepth; @@ -26,23 +29,29 @@ typedef struct { int i_offset; } _PyInstruction; -typedef struct { +typedef struct instruction_sequence { + PyObject_HEAD _PyInstruction *s_instrs; int s_allocated; int s_used; int s_next_free_label; /* next free label id */ + /* Map of a label id to instruction offset (index into s_instrs). * If s_labelmap is NULL, then each label id is the offset itself. */ int *s_labelmap; /* label id --> instr offset */ int s_labelmap_size; + + /* PyList of instruction sequences of nested functions */ + PyObject *s_nested; } _PyInstructionSequence; typedef struct { int id; } _PyJumpTargetLabel; +PyAPI_FUNC(PyObject*)PyInstructionSequence_New(void); int _PyInstructionSequence_UseLabel(_PyInstructionSequence *seq, int lbl); int _PyInstructionSequence_Addop(_PyInstructionSequence *seq, int opcode, int oparg, @@ -53,6 +62,8 @@ int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int po int opcode, int oparg, _Py_SourceLocation loc); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); +PyTypeObject PyInstructionSequence_Type; +#define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &PyInstructionSequence_Type) #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index d75f0f88656128..16a11b01f85915 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -861,6 +861,7 @@ extern "C" { INIT_ID(co_stacksize), \ INIT_ID(co_varnames), \ INIT_ID(code), \ + INIT_ID(col_offset), \ INIT_ID(command), \ INIT_ID(comment_factory), \ INIT_ID(compile_mode), \ @@ -910,6 +911,7 @@ extern "C" { INIT_ID(encode), \ INIT_ID(encoding), \ INIT_ID(end), \ + INIT_ID(end_col_offset), \ INIT_ID(end_lineno), \ INIT_ID(end_offset), \ INIT_ID(endpos), \ @@ -1030,6 +1032,7 @@ extern "C" { INIT_ID(kw1), \ INIT_ID(kw2), \ INIT_ID(kwdefaults), \ + INIT_ID(label), \ INIT_ID(lambda), \ INIT_ID(last), \ INIT_ID(last_exc), \ @@ -1093,6 +1096,7 @@ extern "C" { INIT_ID(namespaces), \ INIT_ID(narg), \ INIT_ID(ndigits), \ + INIT_ID(nested), \ INIT_ID(new_file_name), \ INIT_ID(new_limit), \ INIT_ID(newline), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 7f67e67f571eae..67483edd380a14 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -897,6 +897,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(code); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(col_offset); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(command); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1044,6 +1047,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(end); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(end_col_offset); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(end_lineno); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1404,6 +1410,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(kwdefaults); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(label); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(lambda); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1593,6 +1602,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(ndigits); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(nested); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(new_file_name); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 7a0e884ccc122a..4413cbbd34623c 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,6 +3,7 @@ import unittest import dis import io +import opcode try: import _testinternalcapi except ImportError: @@ -68,16 +69,17 @@ class CompilationStepTestCase(unittest.TestCase): class Label: pass - def assertInstructionsMatch(self, actual_, expected_): - # get two lists where each entry is a label or - # an instruction tuple. Normalize the labels to the - # instruction count of the target, and compare the lists. + def assertInstructionsMatch(self, actual_seq, expected_): + # get an InstructionSequence and an expected list, where each + # entry is a label or an instruction tuple. Construct an expcted + # instruction sequence and compare with the one given. - self.assertIsInstance(actual_, list) self.assertIsInstance(expected_, list) + expected_seq = self.seq_from_insts(expected_) + self.assertIsInstance(actual_seq, type(expected_seq)) - actual = self.normalize_insts(actual_) - expected = self.normalize_insts(expected_) + actual = actual_seq.get_instructions() + expected = expected_seq.get_instructions() self.assertEqual(len(actual), len(expected)) # compare instructions @@ -87,6 +89,8 @@ def assertInstructionsMatch(self, actual_, expected_): continue self.assertIsInstance(exp, tuple) self.assertIsInstance(act, tuple) + while exp[-1] == -1: + exp = exp[:-1] # crop comparison to the provided expected values if len(act) > len(exp): act = act[:len(exp)] @@ -105,6 +109,25 @@ def resolveAndRemoveLabels(self, insts): return res + def seq_from_insts(self, insts): + labels = {item for item in insts if isinstance(item, self.Label)} + for i, lbl in enumerate(labels): + lbl.value = i + + seq = _testinternalcapi.new_instruction_sequence() + for item in insts: + if isinstance(item, self.Label): + seq.use_label(item.value) + else: + op, arg, *loc = item + if isinstance(arg, self.Label): + arg = arg.value + elif arg is None: + arg = 0 + loc = loc + [-1] * (4 - len(loc)) + seq.addop(opcode.opmap[op], arg, *loc) + return seq + def normalize_insts(self, insts): """ Map labels to instruction index. Map opcodes to opnames. diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 9d5f721806a884..3dbe089ef6bc56 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -2,6 +2,7 @@ import dis import io import math +import opcode import os import unittest import sys @@ -11,6 +12,8 @@ import types import textwrap import warnings +import _testinternalcapi + from test import support from test.support import (script_helper, requires_debug_ranges, requires_specialization, Py_C_RECURSION_LIMIT) @@ -2418,6 +2421,45 @@ def test_return_inside_async_with_block(self): """ self.check_stack_size(snippet, async_=True) +class TestInstructionSequence(unittest.TestCase): + def compare_instructions(self, seq, expected): + self.assertEqual([(opcode.opname[i[0]],) + i[1:] for i in seq.get_instructions()], + expected) + + def test_basics(self): + seq = _testinternalcapi.new_instruction_sequence() + seq.addop(opcode.opmap['LOAD_CONST'], 1, 1, 0, 0, 0) + seq.addop(opcode.opmap['JUMP'], lbl1 := seq.new_label(), 2, 0, 0, 0) + seq.addop(opcode.opmap['LOAD_CONST'], 1, 3, 0, 0, 0) + seq.addop(opcode.opmap['JUMP'], lbl2 := seq.new_label(), 4, 0, 0, 0) + seq.use_label(lbl1) + seq.addop(opcode.opmap['LOAD_CONST'], 2, 4, 0, 0, 0) + seq.use_label(lbl2) + seq.addop(opcode.opmap['RETURN_VALUE'], 0, 3, 0, 0, 0) + + expected = [('LOAD_CONST', 1, 1, 0, 0, 0), + ('JUMP', 4, 2, 0, 0, 0), + ('LOAD_CONST', 1, 3, 0, 0, 0), + ('JUMP', 5, 4, 0, 0, 0), + ('LOAD_CONST', 2, 4, 0, 0, 0), + ('RETURN_VALUE', 0, 3, 0, 0, 0), + ] + + self.compare_instructions(seq, expected) + + def test_nested(self): + seq = _testinternalcapi.new_instruction_sequence() + seq.addop(opcode.opmap['LOAD_CONST'], 1, 1, 0, 0, 0) + nested = _testinternalcapi.new_instruction_sequence() + nested.addop(opcode.opmap['LOAD_CONST'], 2, 2, 0, 0, 0) + + self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) + self.compare_instructions(nested, [('LOAD_CONST', 2, 2, 0, 0, 0)]) + + seq.add_nested(nested) + self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) + self.compare_instructions(seq.get_nested()[0], [('LOAD_CONST', 2, 2, 0, 0, 0)]) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 758e88e288bac6..350e9940e5a94b 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -22,6 +22,7 @@ #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() +#include "pycore_instruction_sequence.h" // PyInstructionSequence_New() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() #include "pycore_object.h" // _PyObject_IsFreed() @@ -723,6 +724,19 @@ _testinternalcapi_compiler_cleandoc_impl(PyObject *module, PyObject *doc) return _PyCompile_CleanDoc(doc); } +/*[clinic input] + +_testinternalcapi.new_instruction_sequence -> object + +Return a new, empty InstructionSequence. +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_new_instruction_sequence_impl(PyObject *module) +/*[clinic end generated code: output=ea4243fddb9057fd input=1dec2591b173be83]*/ +{ + return PyInstructionSequence_New(); +} /*[clinic input] @@ -1717,6 +1731,7 @@ static PyMethodDef module_functions[] = { {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL}, {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, _TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF + _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index cba2a943d03456..a61858561d5ef8 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -67,6 +67,24 @@ _testinternalcapi_compiler_cleandoc(PyObject *module, PyObject *const *args, Py_ return return_value; } +PyDoc_STRVAR(_testinternalcapi_new_instruction_sequence__doc__, +"new_instruction_sequence($module, /)\n" +"--\n" +"\n" +"Return a new, empty InstructionSequence."); + +#define _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF \ + {"new_instruction_sequence", (PyCFunction)_testinternalcapi_new_instruction_sequence, METH_NOARGS, _testinternalcapi_new_instruction_sequence__doc__}, + +static PyObject * +_testinternalcapi_new_instruction_sequence_impl(PyObject *module); + +static PyObject * +_testinternalcapi_new_instruction_sequence(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testinternalcapi_new_instruction_sequence_impl(module); +} + PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, "compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" @@ -282,4 +300,4 @@ _testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignore { return _testinternalcapi_test_long_numbits_impl(module); } -/*[clinic end generated code: output=679bf53bbae20085 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9804015d77d36c77 input=a9049054013a1b77]*/ diff --git a/Objects/object.c b/Objects/object.c index 60642d899bcafa..603ed42ddfe527 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -10,6 +10,7 @@ #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_instruction_sequence.h" // PyInstructionSequence_Type #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type @@ -2226,6 +2227,7 @@ static PyTypeObject* static_types[] = { &PyGen_Type, &PyGetSetDescr_Type, &PyInstanceMethod_Type, + &PyInstructionSequence_Type, &PyListIter_Type, &PyListRevIter_Type, &PyList_Type, diff --git a/Python/clinic/instruction_sequence.c.h b/Python/clinic/instruction_sequence.c.h new file mode 100644 index 00000000000000..10ce6c8d37da5e --- /dev/null +++ b/Python/clinic/instruction_sequence.c.h @@ -0,0 +1,304 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_NoKeywords() + +PyDoc_STRVAR(inst_seq_new__doc__, +"InstructionSequenceType()\n" +"--\n" +"\n" +"Create a new InstructionSequence object."); + +static PyObject * +inst_seq_new_impl(PyTypeObject *type); + +static PyObject * +inst_seq_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PyInstructionSequence_Type; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("InstructionSequenceType", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("InstructionSequenceType", kwargs)) { + goto exit; + } + return_value = inst_seq_new_impl(type); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_use_label__doc__, +"use_label($self, /, label)\n" +"--\n" +"\n" +"Place label at current location."); + +#define INSTRUCTIONSEQUENCETYPE_USE_LABEL_METHODDEF \ + {"use_label", _PyCFunction_CAST(InstructionSequenceType_use_label), METH_FASTCALL|METH_KEYWORDS, InstructionSequenceType_use_label__doc__}, + +static PyObject * +InstructionSequenceType_use_label_impl(_PyInstructionSequence *self, + int label); + +static PyObject * +InstructionSequenceType_use_label(_PyInstructionSequence *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(label), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"label", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "use_label", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int label; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + label = PyLong_AsInt(args[0]); + if (label == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = InstructionSequenceType_use_label_impl(self, label); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_addop__doc__, +"addop($self, /, opcode, oparg, lineno, col_offset, end_lineno,\n" +" end_col_offset)\n" +"--\n" +"\n" +"Append an instruction."); + +#define INSTRUCTIONSEQUENCETYPE_ADDOP_METHODDEF \ + {"addop", _PyCFunction_CAST(InstructionSequenceType_addop), METH_FASTCALL|METH_KEYWORDS, InstructionSequenceType_addop__doc__}, + +static PyObject * +InstructionSequenceType_addop_impl(_PyInstructionSequence *self, int opcode, + int oparg, int lineno, int col_offset, + int end_lineno, int end_col_offset); + +static PyObject * +InstructionSequenceType_addop(_PyInstructionSequence *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), &_Py_ID(oparg), &_Py_ID(lineno), &_Py_ID(col_offset), &_Py_ID(end_lineno), &_Py_ID(end_col_offset), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", "oparg", "lineno", "col_offset", "end_lineno", "end_col_offset", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "addop", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + int opcode; + int oparg; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + oparg = PyLong_AsInt(args[1]); + if (oparg == -1 && PyErr_Occurred()) { + goto exit; + } + lineno = PyLong_AsInt(args[2]); + if (lineno == -1 && PyErr_Occurred()) { + goto exit; + } + col_offset = PyLong_AsInt(args[3]); + if (col_offset == -1 && PyErr_Occurred()) { + goto exit; + } + end_lineno = PyLong_AsInt(args[4]); + if (end_lineno == -1 && PyErr_Occurred()) { + goto exit; + } + end_col_offset = PyLong_AsInt(args[5]); + if (end_col_offset == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = InstructionSequenceType_addop_impl(self, opcode, oparg, lineno, col_offset, end_lineno, end_col_offset); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_new_label__doc__, +"new_label($self, /)\n" +"--\n" +"\n" +"Return a new label."); + +#define INSTRUCTIONSEQUENCETYPE_NEW_LABEL_METHODDEF \ + {"new_label", (PyCFunction)InstructionSequenceType_new_label, METH_NOARGS, InstructionSequenceType_new_label__doc__}, + +static int +InstructionSequenceType_new_label_impl(_PyInstructionSequence *self); + +static PyObject * +InstructionSequenceType_new_label(_PyInstructionSequence *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = InstructionSequenceType_new_label_impl(self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_add_nested__doc__, +"add_nested($self, /, nested)\n" +"--\n" +"\n" +"Add a nested sequence."); + +#define INSTRUCTIONSEQUENCETYPE_ADD_NESTED_METHODDEF \ + {"add_nested", _PyCFunction_CAST(InstructionSequenceType_add_nested), METH_FASTCALL|METH_KEYWORDS, InstructionSequenceType_add_nested__doc__}, + +static PyObject * +InstructionSequenceType_add_nested_impl(_PyInstructionSequence *self, + PyObject *nested); + +static PyObject * +InstructionSequenceType_add_nested(_PyInstructionSequence *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(nested), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"nested", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "add_nested", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *nested; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + nested = args[0]; + return_value = InstructionSequenceType_add_nested_impl(self, nested); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_get_nested__doc__, +"get_nested($self, /)\n" +"--\n" +"\n" +"Add a nested sequence."); + +#define INSTRUCTIONSEQUENCETYPE_GET_NESTED_METHODDEF \ + {"get_nested", (PyCFunction)InstructionSequenceType_get_nested, METH_NOARGS, InstructionSequenceType_get_nested__doc__}, + +static PyObject * +InstructionSequenceType_get_nested_impl(_PyInstructionSequence *self); + +static PyObject * +InstructionSequenceType_get_nested(_PyInstructionSequence *self, PyObject *Py_UNUSED(ignored)) +{ + return InstructionSequenceType_get_nested_impl(self); +} + +PyDoc_STRVAR(InstructionSequenceType_get_instructions__doc__, +"get_instructions($self, /)\n" +"--\n" +"\n" +"Return the instructions as a list of tuples or labels."); + +#define INSTRUCTIONSEQUENCETYPE_GET_INSTRUCTIONS_METHODDEF \ + {"get_instructions", (PyCFunction)InstructionSequenceType_get_instructions, METH_NOARGS, InstructionSequenceType_get_instructions__doc__}, + +static PyObject * +InstructionSequenceType_get_instructions_impl(_PyInstructionSequence *self); + +static PyObject * +InstructionSequenceType_get_instructions(_PyInstructionSequence *self, PyObject *Py_UNUSED(ignored)) +{ + return InstructionSequenceType_get_instructions_impl(self); +} +/*[clinic end generated code: output=530661b1c7820520 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 1e8f97e72cdff6..d0a0476a25afd0 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -32,6 +32,7 @@ #include "pycore_code.h" // _PyCode_New() #include "pycore_compile.h" #include "pycore_flowgraph.h" +#include "pycore_instruction_sequence.h" // PyInstructionSequence_New() #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_pystate.h" // _Py_GetConfig() @@ -248,7 +249,7 @@ struct compiler_unit { PyObject *u_private; /* for private name mangling */ PyObject *u_static_attributes; /* for class: attributes accessed via self.X */ - instr_sequence u_instr_sequence; /* codegen output */ + instr_sequence *u_instr_sequence; /* codegen output */ int u_nfblocks; int u_in_inlined_comp; @@ -281,12 +282,12 @@ struct compiler { int c_nestlevel; PyObject *c_const_cache; /* Python dict holding all constants, including names tuple */ - struct compiler_unit *u; /* compiler state for current block */ + struct compiler_unit *u; /* compiler state for current block */ PyObject *c_stack; /* Python list holding compiler_unit ptrs */ PyArena *c_arena; /* pointer to memory allocation arena */ }; -#define INSTR_SEQUENCE(C) (&((C)->u->u_instr_sequence)) +#define INSTR_SEQUENCE(C) ((C)->u->u_instr_sequence) typedef struct { @@ -567,7 +568,7 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) static void compiler_unit_free(struct compiler_unit *u) { - PyInstructionSequence_Fini(&u->u_instr_sequence); + Py_XDECREF(u->u_instr_sequence); Py_CLEAR(u->u_ste); Py_CLEAR(u->u_metadata.u_name); Py_CLEAR(u->u_metadata.u_qualname); @@ -976,7 +977,7 @@ compiler_addop_load_const(PyObject *const_cache, struct compiler_unit *u, locati if (arg < 0) { return ERROR; } - return codegen_addop_i(&u->u_instr_sequence, LOAD_CONST, arg, loc); + return codegen_addop_i(u->u_instr_sequence, LOAD_CONST, arg, loc); } static int @@ -987,7 +988,7 @@ compiler_addop_o(struct compiler_unit *u, location loc, if (arg < 0) { return ERROR; } - return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); + return codegen_addop_i(u->u_instr_sequence, opcode, arg, loc); } static int @@ -1033,7 +1034,7 @@ compiler_addop_name(struct compiler_unit *u, location loc, arg <<= 2; arg |= 1; } - return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); + return codegen_addop_i(u->u_instr_sequence, opcode, arg, loc); } /* Add an opcode with an integer argument */ @@ -1252,6 +1253,8 @@ compiler_enter_scope(struct compiler *c, identifier name, u->u_static_attributes = NULL; } + u->u_instr_sequence = (instr_sequence*)PyInstructionSequence_New(); + /* Push the old compiler_unit on the stack. */ if (c->u) { PyObject *capsule = PyCapsule_New(c->u, CAPSULE_NAME, NULL); @@ -7526,7 +7529,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, if (consts == NULL) { goto error; } - g = instr_sequence_to_cfg(&u->u_instr_sequence); + g = instr_sequence_to_cfg(u->u_instr_sequence); if (g == NULL) { goto error; } @@ -7701,50 +7704,17 @@ instructions_to_cfg(PyObject *instructions) return NULL; } -static PyObject * -instr_sequence_to_instructions(instr_sequence *seq) -{ - PyObject *instructions = PyList_New(0); - if (instructions == NULL) { - return NULL; - } - for (int i = 0; i < seq->s_used; i++) { - instruction *instr = &seq->s_instrs[i]; - location loc = instr->i_loc; - PyObject *inst_tuple = Py_BuildValue( - "(iiiiii)", instr->i_opcode, instr->i_oparg, - loc.lineno, loc.end_lineno, - loc.col_offset, loc.end_col_offset); - if (inst_tuple == NULL) { - goto error; - } - - int res = PyList_Append(instructions, inst_tuple); - Py_DECREF(inst_tuple); - if (res != 0) { - goto error; - } - } - return instructions; -error: - Py_XDECREF(instructions); - return NULL; -} - static PyObject * cfg_to_instructions(cfg_builder *g) { - instr_sequence seq; - memset(&seq, 0, sizeof(seq)); - if (_PyCfg_ToInstructionSequence(g, &seq) < 0) { + instr_sequence *seq = (instr_sequence *)PyInstructionSequence_New(); + if (_PyCfg_ToInstructionSequence(g, seq) < 0) { return NULL; } - if (_PyInstructionSequence_ApplyLabelMap(&seq) < 0) { + if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { return NULL; } - PyObject *res = instr_sequence_to_instructions(&seq); - PyInstructionSequence_Fini(&seq); - return res; + return (PyObject*)seq; } // C implementation of inspect.cleandoc() @@ -7916,13 +7886,8 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, if (_PyInstructionSequence_ApplyLabelMap(INSTR_SEQUENCE(c)) < 0) { return NULL; } - - PyObject *insts = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); - if (insts == NULL) { - goto finally; - } - res = PyTuple_Pack(2, insts, metadata); - Py_DECREF(insts); + /* Allocate a copy of the instruction sequence on the heap */ + res = PyTuple_Pack(2, INSTR_SEQUENCE(c), metadata); finally: Py_XDECREF(metadata); diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index 597d2b73d19f30..1d7737f177186a 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -141,6 +141,21 @@ _PyInstructionSequence_InsertInstruction(instr_sequence *seq, int pos, return SUCCESS; } +int +_PyInstructionSequence_AddNested(instr_sequence *seq, instr_sequence *nested) +{ + if (seq->s_nested == NULL) { + seq->s_nested = PyList_New(0); + if (seq->s_nested == NULL) { + return ERROR; + } + } + if (PyList_Append(seq->s_nested, (PyObject*)nested) < 0) { + return ERROR; + } + return SUCCESS; +} + void PyInstructionSequence_Fini(instr_sequence *seq) { PyMem_Free(seq->s_labelmap); @@ -149,3 +164,275 @@ PyInstructionSequence_Fini(instr_sequence *seq) { PyMem_Free(seq->s_instrs); seq->s_instrs = NULL; } + +/*[clinic input] +class InstructionSequenceType "_PyInstructionSequence *" "&PyInstructionSequence_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=62b593e603622574]*/ + +#include "clinic/instruction_sequence.c.h" + +static _PyInstructionSequence* +inst_seq_create_raw(void) +{ + _PyInstructionSequence *seq; + seq = PyObject_GC_New(_PyInstructionSequence, &PyInstructionSequence_Type); + seq->s_instrs = NULL; + seq->s_allocated = 0; + seq->s_used = 0; + seq->s_next_free_label = 0; + seq->s_labelmap = NULL; + seq->s_labelmap_size = 0; + seq->s_nested = NULL; + + PyObject_GC_Track(seq); + return seq; +} + +PyObject* +PyInstructionSequence_New(void) +{ + _PyInstructionSequence *seq = inst_seq_create_raw(); + if (seq == NULL) { + return NULL; + } + return (PyObject*)seq; +} + +/*[clinic input] +@classmethod +InstructionSequenceType.__new__ as inst_seq_new + +Create a new InstructionSequence object. +[clinic start generated code]*/ + +static PyObject * +inst_seq_new_impl(PyTypeObject *type) +/*[clinic end generated code: output=98881de92c8876f6 input=b393150146849c74]*/ +{ + return (PyObject*)inst_seq_create_raw(); +} + +/*[clinic input] +InstructionSequenceType.use_label + + label: int + +Place label at current location. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_use_label_impl(_PyInstructionSequence *self, + int label) +/*[clinic end generated code: output=4c06bbacb2854755 input=da55f49bb91841f3]*/ + +{ + if (_PyInstructionSequence_UseLabel(self, label) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.addop + + opcode: int + oparg: int + lineno: int + col_offset: int + end_lineno: int + end_col_offset: int + +Append an instruction. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_addop_impl(_PyInstructionSequence *self, int opcode, + int oparg, int lineno, int col_offset, + int end_lineno, int end_col_offset) +/*[clinic end generated code: output=af0cc22c048dfbf3 input=012762ac88198713]*/ +{ + _Py_SourceLocation loc = {lineno, col_offset, end_lineno, end_col_offset}; + if (_PyInstructionSequence_Addop(self, opcode, oparg, loc) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.new_label -> int + +Return a new label. +[clinic start generated code]*/ + +static int +InstructionSequenceType_new_label_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=dcb0589e4f5bf4bd input=c66040b9897bc327]*/ +{ + _PyJumpTargetLabel lbl = _PyInstructionSequence_NewLabel(self); + return lbl.id; +} + +/*[clinic input] +InstructionSequenceType.add_nested + + nested: object + +Add a nested sequence. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_add_nested_impl(_PyInstructionSequence *self, + PyObject *nested) +/*[clinic end generated code: output=14540fad459f7971 input=f2c482568b3b3c0f]*/ +{ + if (!_PyInstructionSequence_Check(nested)) { + PyErr_Format(PyExc_TypeError, + "expected an instruction sequence, not '%.100s'", + Py_TYPE(nested)->tp_name); + return NULL; + } + if (_PyInstructionSequence_AddNested(self, (_PyInstructionSequence*)nested) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.get_nested + +Add a nested sequence. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_get_nested_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=f415112c292630cb input=e429e474c57b95b4]*/ +{ + if (self->s_nested == NULL) { + return PyList_New(0); + } + return Py_NewRef(self->s_nested); +} + +/*[clinic input] +InstructionSequenceType.get_instructions + +Return the instructions as a list of tuples or labels. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_get_instructions_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=23f4f3f894c301b3 input=fbadb5dadb611291]*/ +{ + if (_PyInstructionSequence_ApplyLabelMap(self) < 0) { + return NULL; + } + PyObject *instructions = PyList_New(0); + if (instructions == NULL) { + return NULL; + } + for (int i = 0; i < self->s_used; i++) { + instruction *instr = &self->s_instrs[i]; + location loc = instr->i_loc; + PyObject *inst_tuple = Py_BuildValue( + "(iiiiii)", instr->i_opcode, instr->i_oparg, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + if (inst_tuple == NULL) { + goto error; + } + + int res = PyList_Append(instructions, inst_tuple); + Py_DECREF(inst_tuple); + if (res != 0) { + goto error; + } + } + return instructions; +error: + Py_XDECREF(instructions); + return NULL; +} + +static PyMethodDef inst_seq_methods[] = { + INSTRUCTIONSEQUENCETYPE_ADDOP_METHODDEF + INSTRUCTIONSEQUENCETYPE_NEW_LABEL_METHODDEF + INSTRUCTIONSEQUENCETYPE_USE_LABEL_METHODDEF + INSTRUCTIONSEQUENCETYPE_ADD_NESTED_METHODDEF + INSTRUCTIONSEQUENCETYPE_GET_NESTED_METHODDEF + INSTRUCTIONSEQUENCETYPE_GET_INSTRUCTIONS_METHODDEF + {NULL, NULL, 0, NULL}, +}; + +static PyMemberDef inst_seq_memberlist[] = { + {NULL} /* Sentinel */ +}; + +static PyGetSetDef inst_seq_getsetters[] = { + {NULL} /* Sentinel */ +}; + +static void +inst_seq_dealloc(_PyInstructionSequence *seq) +{ + PyObject_GC_UnTrack(seq); + Py_TRASHCAN_BEGIN(seq, inst_seq_dealloc) + PyInstructionSequence_Fini(seq); + PyObject_GC_Del(seq); + Py_TRASHCAN_END +} + +static int +inst_seq_traverse(_PyInstructionSequence *seq, visitproc visit, void *arg) +{ + Py_VISIT(seq->s_nested); + return 0; +} + +static int +inst_seq_clear(_PyInstructionSequence *seq) +{ + Py_CLEAR(seq->s_nested); + return 0; +} + +PyTypeObject PyInstructionSequence_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "InstructionSequence", + sizeof(_PyInstructionSequence), + 0, + (destructor)inst_seq_dealloc, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + inst_seq_new__doc__, /* tp_doc */ + (traverseproc)inst_seq_traverse, /* tp_traverse */ + (inquiry)inst_seq_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + inst_seq_methods, /* tp_methods */ + inst_seq_memberlist, /* tp_members */ + inst_seq_getsetters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + inst_seq_new, /* tp_new */ +}; From 7414dfbcc88756d350650b633e6372e543e6b438 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 8 Apr 2024 16:23:54 +0100 Subject: [PATCH 02/15] PyInstructionSequence_New --> _PyInstructionSequence_New --- Include/internal/pycore_instruction_sequence.h | 3 ++- Modules/_testinternalcapi.c | 4 ++-- Python/compile.c | 6 +++--- Python/instruction_sequence.c | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index e6d25bbdfad3c7..88af26b8e01b4d 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -51,7 +51,8 @@ typedef struct { int id; } _PyJumpTargetLabel; -PyAPI_FUNC(PyObject*)PyInstructionSequence_New(void); +PyAPI_FUNC(PyObject*)_PyInstructionSequence_New(void); + int _PyInstructionSequence_UseLabel(_PyInstructionSequence *seq, int lbl); int _PyInstructionSequence_Addop(_PyInstructionSequence *seq, int opcode, int oparg, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 350e9940e5a94b..d32fb8f2b2713c 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -22,7 +22,7 @@ #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() -#include "pycore_instruction_sequence.h" // PyInstructionSequence_New() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_New() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() #include "pycore_object.h" // _PyObject_IsFreed() @@ -735,7 +735,7 @@ static PyObject * _testinternalcapi_new_instruction_sequence_impl(PyObject *module) /*[clinic end generated code: output=ea4243fddb9057fd input=1dec2591b173be83]*/ { - return PyInstructionSequence_New(); + return _PyInstructionSequence_New(); } /*[clinic input] diff --git a/Python/compile.c b/Python/compile.c index d0a0476a25afd0..4ba649be86a48d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -32,7 +32,7 @@ #include "pycore_code.h" // _PyCode_New() #include "pycore_compile.h" #include "pycore_flowgraph.h" -#include "pycore_instruction_sequence.h" // PyInstructionSequence_New() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_New() #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_pystate.h" // _Py_GetConfig() @@ -1253,7 +1253,7 @@ compiler_enter_scope(struct compiler *c, identifier name, u->u_static_attributes = NULL; } - u->u_instr_sequence = (instr_sequence*)PyInstructionSequence_New(); + u->u_instr_sequence = (instr_sequence*)_PyInstructionSequence_New(); /* Push the old compiler_unit on the stack. */ if (c->u) { @@ -7707,7 +7707,7 @@ instructions_to_cfg(PyObject *instructions) static PyObject * cfg_to_instructions(cfg_builder *g) { - instr_sequence *seq = (instr_sequence *)PyInstructionSequence_New(); + instr_sequence *seq = (instr_sequence *)_PyInstructionSequence_New(); if (_PyCfg_ToInstructionSequence(g, seq) < 0) { return NULL; } diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index 1d7737f177186a..c3eba3b7c4abcb 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -190,7 +190,7 @@ inst_seq_create_raw(void) } PyObject* -PyInstructionSequence_New(void) +_PyInstructionSequence_New(void) { _PyInstructionSequence *seq = inst_seq_create_raw(); if (seq == NULL) { From 00f459e576feebe04b873ba8ced8c905b8b57798 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 8 Apr 2024 16:34:26 +0100 Subject: [PATCH 03/15] remove redundant comment --- Include/internal/pycore_instruction_sequence.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index 88af26b8e01b4d..e103299850ef04 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -40,7 +40,7 @@ typedef struct instruction_sequence { /* Map of a label id to instruction offset (index into s_instrs). * If s_labelmap is NULL, then each label id is the offset itself. */ - int *s_labelmap; /* label id --> instr offset */ + int *s_labelmap; int s_labelmap_size; /* PyList of instruction sequences of nested functions */ From ebf19fa1be347ae80836a1736dd1d60755c6351a Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 9 Apr 2024 00:54:46 +0100 Subject: [PATCH 04/15] simplify tests --- Lib/test/support/bytecode_helper.py | 35 ++++++++++++++++------------- Lib/test/test_peepholer.py | 1 - 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 4413cbbd34623c..fac7cde9ef926e 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -69,17 +69,13 @@ class CompilationStepTestCase(unittest.TestCase): class Label: pass - def assertInstructionsMatch(self, actual_seq, expected_): + def assertInstructionsMatch(self, actual_seq, expected): # get an InstructionSequence and an expected list, where each # entry is a label or an instruction tuple. Construct an expcted # instruction sequence and compare with the one given. - self.assertIsInstance(expected_, list) - expected_seq = self.seq_from_insts(expected_) - self.assertIsInstance(actual_seq, type(expected_seq)) - + self.assertIsInstance(expected, list) actual = actual_seq.get_instructions() - expected = expected_seq.get_instructions() self.assertEqual(len(actual), len(expected)) # compare instructions @@ -89,12 +85,16 @@ def assertInstructionsMatch(self, actual_seq, expected_): continue self.assertIsInstance(exp, tuple) self.assertIsInstance(act, tuple) - while exp[-1] == -1: - exp = exp[:-1] - # crop comparison to the provided expected values - if len(act) > len(exp): - act = act[:len(exp)] - self.assertEqual(exp, act) + exp, act = list(reversed(exp)), list(reversed(act)) + op1, op2 = exp.pop(), opcode.opname[act.pop()] + self.assertEqual(op1, op2, "op mismatch") + if opcode.opmap[op1] in self.HAS_ARG: + arg1, arg2 = exp.pop(), act.pop() + self.assertEqual(arg1, arg2, "arg mismatch") + else: + act.pop() + loc1, loc2 = exp, act + self.assertEqual(loc1, loc2[-len(loc1):], "location mismatch") def resolveAndRemoveLabels(self, insts): idx = 0 @@ -119,11 +119,14 @@ def seq_from_insts(self, insts): if isinstance(item, self.Label): seq.use_label(item.value) else: - op, arg, *loc = item - if isinstance(arg, self.Label): - arg = arg.value - elif arg is None: + op = item[0] + if op in self.HAS_ARG: + arg, *loc = item[1:] + if isinstance(arg, self.Label): + arg = arg.value + else: arg = 0 + loc = list(item[1:]) loc = loc + [-1] * (4 - len(loc)) seq.addop(opcode.opmap[op], arg, *loc) return seq diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index dffedd0b1fc476..5fab39d7d85f5d 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -984,7 +984,6 @@ def cfg_optimization_test(self, insts, expected_insts, if expected_consts is None: expected_consts = consts opt_insts, opt_consts = self.get_optimized(insts, consts, nlocals) - expected_insts = self.normalize_insts(expected_insts) self.assertInstructionsMatch(opt_insts, expected_insts) self.assertEqual(opt_consts, expected_consts) From a413ee1d8bc07e9d872bf8ae36838e82bf17c791 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 9 Apr 2024 15:47:20 +0100 Subject: [PATCH 05/15] use instruction sequence in tests --- Lib/test/support/bytecode_helper.py | 73 +++++----------- Lib/test/test_compiler_assemble.py | 20 ++--- Lib/test/test_peepholer.py | 47 +++++----- Python/compile.c | 130 ++++------------------------ Python/instruction_sequence.c | 18 +++- 5 files changed, 88 insertions(+), 200 deletions(-) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index fac7cde9ef926e..85bcd1f0f1cd4f 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -76,6 +76,7 @@ def assertInstructionsMatch(self, actual_seq, expected): self.assertIsInstance(expected, list) actual = actual_seq.get_instructions() + expected = self.seq_from_insts(expected).get_instructions() self.assertEqual(len(actual), len(expected)) # compare instructions @@ -85,16 +86,8 @@ def assertInstructionsMatch(self, actual_seq, expected): continue self.assertIsInstance(exp, tuple) self.assertIsInstance(act, tuple) - exp, act = list(reversed(exp)), list(reversed(act)) - op1, op2 = exp.pop(), opcode.opname[act.pop()] - self.assertEqual(op1, op2, "op mismatch") - if opcode.opmap[op1] in self.HAS_ARG: - arg1, arg2 = exp.pop(), act.pop() - self.assertEqual(arg1, arg2, "arg mismatch") - else: - act.pop() - loc1, loc2 = exp, act - self.assertEqual(loc1, loc2[-len(loc1):], "location mismatch") + idx = max([p[0] for p in enumerate(exp) if p[1] != -1]) + self.assertEqual(exp[:idx], act[:idx]) def resolveAndRemoveLabels(self, insts): idx = 0 @@ -120,46 +113,26 @@ def seq_from_insts(self, insts): seq.use_label(item.value) else: op = item[0] - if op in self.HAS_ARG: - arg, *loc = item[1:] - if isinstance(arg, self.Label): - arg = arg.value - else: - arg = 0 - loc = list(item[1:]) + if isinstance(op, str): + op = opcode.opmap[op] + arg, *loc = item[1:] + if isinstance(arg, self.Label): + arg = arg.value loc = loc + [-1] * (4 - len(loc)) - seq.addop(opcode.opmap[op], arg, *loc) + seq.addop(op, arg or 0, *loc) return seq - def normalize_insts(self, insts): - """ Map labels to instruction index. - Map opcodes to opnames. - """ - insts = self.resolveAndRemoveLabels(insts) - res = [] - for item in insts: - assert isinstance(item, tuple) - opcode, oparg, *loc = item - opcode = dis.opmap.get(opcode, opcode) - if isinstance(oparg, self.Label): - arg = oparg.value - else: - arg = oparg if opcode in self.HAS_ARG else None - opcode = dis.opname[opcode] - res.append((opcode, arg, *loc)) - return res - - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] - for item in insts: - assert isinstance(item, tuple) - inst = list(item) - opcode = dis.opmap[inst[0]] - oparg = inst[1] - loc = inst[2:] + [-1] * (6 - len(inst)) - res.append((opcode, oparg, *loc)) - return res + def check_instructions(self, insts): + for inst in insts: + if isinstance(inst, self.Label): + continue + op, arg, *loc = inst + if isinstance(op, str): + op = opcode.opmap[op] + self.assertEqual(op in opcode.hasarg, + arg is not None, + f"{opcode.opname[op]=} {arg=}") + self.assertTrue(all(isinstance(l, int) for l in loc)) @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") @@ -173,10 +146,8 @@ def generate_code(self, ast): @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CfgOptimizationTestCase(CompilationStepTestCase): - def get_optimized(self, insts, consts, nlocals=0): - insts = self.normalize_insts(insts) - insts = self.complete_insts_info(insts) - insts = _testinternalcapi.optimize_cfg(insts, consts, nlocals) + def get_optimized(self, seq, consts, nlocals=0): + insts = _testinternalcapi.optimize_cfg(seq, consts, nlocals) return insts, consts @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index ab9f04dd63af20..1b98b0d97ed8a5 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -27,8 +27,8 @@ def complete_metadata(self, metadata, filename="myfile.py"): def insts_to_code_object(self, insts, metadata): metadata = self.complete_metadata(metadata) - insts = self.complete_insts_info(insts) - return self.get_code_object(metadata['filename'], insts, metadata) + seq = self.seq_from_insts(insts) + return self.get_code_object(metadata['filename'], seq, metadata) def assemble_test(self, insts, metadata, expected): co = self.insts_to_code_object(insts, metadata) @@ -71,7 +71,7 @@ def test_simple_expr(self): ('BINARY_OP', 0, 1), # '+' ('LOAD_CONST', 0, 1), # 2 ('BINARY_OP', 11, 1), # '/' - ('RETURN_VALUE', 1), + ('RETURN_VALUE', None, 1), ] expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} self.assemble_test(insts, metadata, expected) @@ -102,13 +102,13 @@ def inner(): ('LOAD_CLOSURE', 0, 1), ('BUILD_TUPLE', 1, 1), ('LOAD_CONST', 1, 1), - ('MAKE_FUNCTION', 0, 2), + ('MAKE_FUNCTION', None, 2), ('SET_FUNCTION_ATTRIBUTE', 8, 2), - ('PUSH_NULL', 0, 1), + ('PUSH_NULL', None, 1), ('CALL', 0, 2), # (lambda: x)() ('LOAD_CONST', 2, 2), # 2 ('BINARY_OP', 6, 2), # % - ('RETURN_VALUE', 0, 2) + ('RETURN_VALUE', None, 2) ] expected = {(0,): 0, (1,): 1, (2,): 0, (120,): 0, (121,): 1} @@ -128,12 +128,12 @@ def test_exception_table(self): ('SETUP_FINALLY', 3), ('RETURN_CONST', 0), ('SETUP_CLEANUP', 8), - ('PUSH_EXC_INFO', 0), - ('POP_TOP', 0), - ('POP_EXCEPT', 0), + ('PUSH_EXC_INFO', None), + ('POP_TOP', None), + ('POP_EXCEPT', None), ('RETURN_CONST', 0), ('COPY', 3), - ('POP_EXCEPT', 0), + ('POP_EXCEPT', None), ('RERAISE', 1), ] co = self.insts_to_code_object(insts, metadata) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 5fab39d7d85f5d..6989aafd293138 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1,5 +1,6 @@ import dis from itertools import combinations, product +import opcode import sys import textwrap import unittest @@ -981,9 +982,15 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): def cfg_optimization_test(self, insts, expected_insts, consts=None, expected_consts=None, nlocals=0): + + self.check_instructions(insts) + self.check_instructions(expected_insts) + if expected_consts is None: expected_consts = consts - opt_insts, opt_consts = self.get_optimized(insts, consts, nlocals) + seq = self.seq_from_insts(insts) + opt_insts, opt_consts = self.get_optimized(seq, consts, nlocals) + expected_insts = self.seq_from_insts(expected_insts).get_instructions() self.assertInstructionsMatch(opt_insts, expected_insts) self.assertEqual(opt_consts, expected_consts) @@ -992,10 +999,10 @@ def test_conditional_jump_forward_non_const_condition(self): ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), ('LOAD_CONST', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), lbl, ('LOAD_CONST', 3, 14), - ('RETURN_VALUE', 14), + ('RETURN_VALUE', None, 14), ] expected_insts = [ ('LOAD_NAME', 1, 11), @@ -1019,11 +1026,11 @@ def test_conditional_jump_forward_const_condition(self): ('LOAD_CONST', 2, 13), lbl, ('LOAD_CONST', 3, 14), - ('RETURN_VALUE', 14), + ('RETURN_VALUE', None, 14), ] expected_insts = [ - ('NOP', 11), - ('NOP', 12), + ('NOP', None, 11), + ('NOP', None, 12), ('RETURN_CONST', 1, 14), ] self.cfg_optimization_test(insts, @@ -1037,14 +1044,14 @@ def test_conditional_jump_backward_non_const_condition(self): ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_NAME', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] expected = [ lbl := self.Label(), ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl, 12), ('LOAD_NAME', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] self.cfg_optimization_test(insts, expected, consts=list(range(5))) @@ -1055,11 +1062,11 @@ def test_conditional_jump_backward_const_condition(self): ('LOAD_CONST', 3, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_CONST', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] expected_insts = [ lbl := self.Label(), - ('NOP', 11), + ('NOP', None, 11), ('JUMP', lbl, 12), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) @@ -1067,7 +1074,7 @@ def test_conditional_jump_backward_const_condition(self): def test_except_handler_label(self): insts = [ ('SETUP_FINALLY', handler := self.Label(), 10), - ('POP_BLOCK', 0, -1), + ('POP_BLOCK', None, -1), ('RETURN_CONST', 1, 11), handler, ('RETURN_CONST', 2, 12), @@ -1089,16 +1096,16 @@ def test_no_unsafe_static_swap(self): ('SWAP', 3, 4), ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 4), - ('POP_TOP', 0, 4), + ('POP_TOP', None, 4), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), ('LOAD_CONST', 1, 2), - ('NOP', 0, 3), + ('NOP', None, 3), ('STORE_FAST', 1, 4), - ('POP_TOP', 0, 4), + ('POP_TOP', None, 4), ('RETURN_CONST', 0) ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) @@ -1112,13 +1119,13 @@ def test_dead_store_elimination_in_same_lineno(self): ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 4), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), ('LOAD_CONST', 1, 2), - ('NOP', 0, 3), - ('POP_TOP', 0, 4), + ('NOP', None, 3), + ('POP_TOP', None, 4), ('STORE_FAST', 1, 4), ('RETURN_CONST', 0, 5) ] @@ -1133,7 +1140,7 @@ def test_no_dead_store_elimination_in_different_lineno(self): ('STORE_FAST', 1, 5), ('STORE_FAST', 1, 6), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), @@ -1168,7 +1175,7 @@ def get_insts(lno1, lno2, op1, op2): op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' expected_insts = [ ('LOAD_NAME', 0, 10), - ('NOP', 0, 4), + ('NOP', None, 4), (op, 0, 5), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) diff --git a/Python/compile.c b/Python/compile.c index 4ba649be86a48d..4df11df630b1d7 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7596,116 +7596,9 @@ optimize_and_assemble(struct compiler *c, int addNone) * a jump target label marking the beginning of a basic block. */ -static int -instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) -{ - assert(PyList_Check(instructions)); - - Py_ssize_t num_insts = PyList_GET_SIZE(instructions); - bool *is_target = PyMem_Calloc(num_insts, sizeof(bool)); - if (is_target == NULL) { - return ERROR; - } - for (Py_ssize_t i = 0; i < num_insts; i++) { - PyObject *item = PyList_GET_ITEM(instructions, i); - if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { - PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); - goto error; - } - int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); - if (PyErr_Occurred()) { - goto error; - } - if (HAS_TARGET(opcode)) { - int oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - goto error; - } - if (oparg < 0 || oparg >= num_insts) { - PyErr_SetString(PyExc_ValueError, "label out of range"); - goto error; - } - is_target[oparg] = true; - } - } - - for (int i = 0; i < num_insts; i++) { - if (is_target[i]) { - if (_PyInstructionSequence_UseLabel(seq, i) < 0) { - goto error; - } - } - PyObject *item = PyList_GET_ITEM(instructions, i); - if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { - PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); - goto error; - } - int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); - if (PyErr_Occurred()) { - goto error; - } - int oparg; - if (OPCODE_HAS_ARG(opcode)) { - oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - goto error; - } - } - else { - oparg = 0; - } - location loc; - loc.lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 2)); - if (PyErr_Occurred()) { - goto error; - } - loc.end_lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 3)); - if (PyErr_Occurred()) { - goto error; - } - loc.col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 4)); - if (PyErr_Occurred()) { - goto error; - } - loc.end_col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 5)); - if (PyErr_Occurred()) { - goto error; - } - if (_PyInstructionSequence_Addop(seq, opcode, oparg, loc) < 0) { - goto error; - } - } - PyMem_Free(is_target); - return SUCCESS; -error: - PyMem_Free(is_target); - return ERROR; -} - -static cfg_builder* -instructions_to_cfg(PyObject *instructions) -{ - cfg_builder *g = NULL; - instr_sequence seq; - memset(&seq, 0, sizeof(instr_sequence)); - - if (instructions_to_instr_sequence(instructions, &seq) < 0) { - goto error; - } - g = instr_sequence_to_cfg(&seq); - if (g == NULL) { - goto error; - } - PyInstructionSequence_Fini(&seq); - return g; -error: - _PyCfgBuilder_Free(g); - PyInstructionSequence_Fini(&seq); - return NULL; -} static PyObject * -cfg_to_instructions(cfg_builder *g) +cfg_to_instruction_sequence(cfg_builder *g) { instr_sequence *seq = (instr_sequence *)_PyInstructionSequence_New(); if (_PyCfg_ToInstructionSequence(g, seq) < 0) { @@ -7898,16 +7791,19 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, } PyObject * -_PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts, int nlocals) +_PyCompile_OptimizeCfg(PyObject *seq, PyObject *consts, int nlocals) { - cfg_builder *g = NULL; - PyObject *res = NULL; + if (!_PyInstructionSequence_Check(seq)) { + PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); + return NULL; + } PyObject *const_cache = PyDict_New(); if (const_cache == NULL) { return NULL; } - g = instructions_to_cfg(instructions); + PyObject *res = NULL; + cfg_builder *g = instr_sequence_to_cfg((instr_sequence*)seq); if (g == NULL) { goto error; } @@ -7916,7 +7812,7 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts, int nlocals) nparams, firstlineno) < 0) { goto error; } - res = cfg_to_instructions(g); + res = cfg_to_instruction_sequence(g); error: Py_DECREF(const_cache); _PyCfgBuilder_Free(g); @@ -7927,8 +7823,12 @@ int _PyCfg_JumpLabelsToTargets(cfg_builder *g); PyCodeObject * _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, - PyObject *instructions) + PyObject *seq) { + if (!_PyInstructionSequence_Check(seq)) { + PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); + return NULL; + } cfg_builder *g = NULL; PyCodeObject *co = NULL; instr_sequence optimized_instrs; @@ -7939,7 +7839,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, return NULL; } - g = instructions_to_cfg(instructions); + g = instr_sequence_to_cfg((instr_sequence*)seq); if (g == NULL) { goto error; } diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index c3eba3b7c4abcb..24e0f885b770b5 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -334,10 +334,20 @@ InstructionSequenceType_get_instructions_impl(_PyInstructionSequence *self) for (int i = 0; i < self->s_used; i++) { instruction *instr = &self->s_instrs[i]; location loc = instr->i_loc; - PyObject *inst_tuple = Py_BuildValue( - "(iiiiii)", instr->i_opcode, instr->i_oparg, - loc.lineno, loc.end_lineno, - loc.col_offset, loc.end_col_offset); + PyObject *inst_tuple; + + if (OPCODE_HAS_ARG(instr->i_opcode)) { + inst_tuple = Py_BuildValue( + "(iiiiii)", instr->i_opcode, instr->i_oparg, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + } + else { + inst_tuple = Py_BuildValue( + "(iOiiii)", instr->i_opcode, Py_None, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + } if (inst_tuple == NULL) { goto error; } From 60e95f89856a1afb7cb0efdc764859ea0f760e78 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 9 Apr 2024 15:59:38 +0100 Subject: [PATCH 06/15] fix test --- Lib/test/test_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 3dbe089ef6bc56..fb991e8da72bd7 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -2442,7 +2442,7 @@ def test_basics(self): ('LOAD_CONST', 1, 3, 0, 0, 0), ('JUMP', 5, 4, 0, 0, 0), ('LOAD_CONST', 2, 4, 0, 0, 0), - ('RETURN_VALUE', 0, 3, 0, 0, 0), + ('RETURN_VALUE', None, 3, 0, 0, 0), ] self.compare_instructions(seq, expected) From ede5bf99b429b51f462ad50088a658c350ce19b6 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 9 Apr 2024 16:07:12 +0100 Subject: [PATCH 07/15] add news --- .../2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst new file mode 100644 index 00000000000000..fd437b7fdd3614 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst @@ -0,0 +1 @@ +Give ``_PyInstructionSequence`` a Python interface and use it in tests. From c78542f769f706b7eb28bde62b6af6599cf5d345 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 9 Apr 2024 17:03:07 +0100 Subject: [PATCH 08/15] extern --- Include/internal/pycore_instruction_sequence.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index e103299850ef04..11a4c7bc9c25a8 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -63,7 +63,7 @@ int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int po int opcode, int oparg, _Py_SourceLocation loc); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); -PyTypeObject PyInstructionSequence_Type; +extern PyTypeObject PyInstructionSequence_Type; #define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &PyInstructionSequence_Type) #ifdef __cplusplus From 9896f59edc4b2811e91ca9fbc8209009782758e8 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 10 Apr 2024 13:06:17 +0100 Subject: [PATCH 09/15] add missing error check Co-authored-by: Erlend E. Aasland --- Python/instruction_sequence.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index 24e0f885b770b5..701c23d92a060f 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -177,6 +177,9 @@ inst_seq_create_raw(void) { _PyInstructionSequence *seq; seq = PyObject_GC_New(_PyInstructionSequence, &PyInstructionSequence_Type); + if (seq == NULL) { + return NULL; + } seq->s_instrs = NULL; seq->s_allocated = 0; seq->s_used = 0; From 78c09289294e38134bf97a0c2eee3343820eb9fa Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 10 Apr 2024 13:06:45 +0100 Subject: [PATCH 10/15] %T Co-authored-by: Erlend E. Aasland --- Python/instruction_sequence.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index 701c23d92a060f..18f5d4493015f5 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -291,8 +291,8 @@ InstructionSequenceType_add_nested_impl(_PyInstructionSequence *self, { if (!_PyInstructionSequence_Check(nested)) { PyErr_Format(PyExc_TypeError, - "expected an instruction sequence, not '%.100s'", - Py_TYPE(nested)->tp_name); + "expected an instruction sequence, not %T", + Py_TYPE(nested)); return NULL; } if (_PyInstructionSequence_AddNested(self, (_PyInstructionSequence*)nested) < 0) { From 54f4111a04619e74d84a6ca5a33a805d161dcc54 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 10 Apr 2024 13:17:00 +0100 Subject: [PATCH 11/15] Py_CLEAR --- Python/compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/compile.c b/Python/compile.c index 4df11df630b1d7..5f9cf47ffb3f60 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -568,7 +568,7 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) static void compiler_unit_free(struct compiler_unit *u) { - Py_XDECREF(u->u_instr_sequence); + Py_CLEAR(u->u_instr_sequence); Py_CLEAR(u->u_ste); Py_CLEAR(u->u_metadata.u_name); Py_CLEAR(u->u_metadata.u_qualname); From 07f108be177c243b3bcb33e40bbd55a2370426b8 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 10 Apr 2024 14:26:03 +0100 Subject: [PATCH 12/15] PyInstructionSequence_Type --> _PyInstructionSequence_Type --- Include/internal/pycore_instruction_sequence.h | 4 ++-- Objects/object.c | 4 ++-- Python/clinic/instruction_sequence.c.h | 4 ++-- Python/instruction_sequence.c | 8 ++++---- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index 11a4c7bc9c25a8..ecba0d9d8e996e 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -63,8 +63,8 @@ int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int po int opcode, int oparg, _Py_SourceLocation loc); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); -extern PyTypeObject PyInstructionSequence_Type; -#define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &PyInstructionSequence_Type) +extern PyTypeObject _PyInstructionSequence_Type; +#define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &_PyInstructionSequence_Type) #ifdef __cplusplus } diff --git a/Objects/object.c b/Objects/object.c index 3000eabab5d738..e3ec751f0e8d3b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -10,7 +10,7 @@ #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() -#include "pycore_instruction_sequence.h" // PyInstructionSequence_Type +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type @@ -2233,7 +2233,6 @@ static PyTypeObject* static_types[] = { &PyGen_Type, &PyGetSetDescr_Type, &PyInstanceMethod_Type, - &PyInstructionSequence_Type, &PyListIter_Type, &PyListRevIter_Type, &PyList_Type, @@ -2286,6 +2285,7 @@ static PyTypeObject* static_types[] = { &_PyHamt_BitmapNode_Type, &_PyHamt_CollisionNode_Type, &_PyHamt_Type, + &_PyInstructionSequence_Type, &_PyLegacyEventHandler_Type, &_PyLineIterator, &_PyManagedBuffer_Type, diff --git a/Python/clinic/instruction_sequence.c.h b/Python/clinic/instruction_sequence.c.h index 10ce6c8d37da5e..66c2ccadfa03c9 100644 --- a/Python/clinic/instruction_sequence.c.h +++ b/Python/clinic/instruction_sequence.c.h @@ -21,7 +21,7 @@ static PyObject * inst_seq_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - PyTypeObject *base_tp = &PyInstructionSequence_Type; + PyTypeObject *base_tp = &_PyInstructionSequence_Type; if ((type == base_tp || type->tp_init == base_tp->tp_init) && !_PyArg_NoPositional("InstructionSequenceType", args)) { @@ -301,4 +301,4 @@ InstructionSequenceType_get_instructions(_PyInstructionSequence *self, PyObject { return InstructionSequenceType_get_instructions_impl(self); } -/*[clinic end generated code: output=530661b1c7820520 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8809d7aa11d9b2bb input=a9049054013a1b77]*/ diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index 18f5d4493015f5..44717b8cd2d88c 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -166,9 +166,9 @@ PyInstructionSequence_Fini(instr_sequence *seq) { } /*[clinic input] -class InstructionSequenceType "_PyInstructionSequence *" "&PyInstructionSequence_Type" +class InstructionSequenceType "_PyInstructionSequence *" "&_PyInstructionSequence_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=62b593e603622574]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=589963e07480390f]*/ #include "clinic/instruction_sequence.c.h" @@ -176,7 +176,7 @@ static _PyInstructionSequence* inst_seq_create_raw(void) { _PyInstructionSequence *seq; - seq = PyObject_GC_New(_PyInstructionSequence, &PyInstructionSequence_Type); + seq = PyObject_GC_New(_PyInstructionSequence, &_PyInstructionSequence_Type); if (seq == NULL) { return NULL; } @@ -409,7 +409,7 @@ inst_seq_clear(_PyInstructionSequence *seq) return 0; } -PyTypeObject PyInstructionSequence_Type = { +PyTypeObject _PyInstructionSequence_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "InstructionSequence", sizeof(_PyInstructionSequence), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 65f94e50e1bd7d..79a8f850ec1451 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -103,6 +103,7 @@ Python/bltinmodule.c - PyZip_Type - Python/context.c - PyContextToken_Type - Python/context.c - PyContextVar_Type - Python/context.c - PyContext_Type - +Python/instruction_sequence.c - _PyInstructionSequence_Type - Python/traceback.c - PyTraceBack_Type - ##----------------------- From 6f8412f3871811159c97a05308e0706b30b5621f Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:27:06 +0100 Subject: [PATCH 13/15] TypeError Co-authored-by: Mark Shannon --- Python/compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/compile.c b/Python/compile.c index 5f9cf47ffb3f60..1f4de1c2128ccf 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7826,7 +7826,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, PyObject *seq) { if (!_PyInstructionSequence_Check(seq)) { - PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); + PyErr_SetString(PyExc_TypeError, "expected an instruction sequence"); return NULL; } cfg_builder *g = NULL; From 9d2f4cc1ebf19c80123e8b445168ac71f4dc29e7 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 17 Apr 2024 11:49:12 +0100 Subject: [PATCH 14/15] rename function. tidy up test --- Lib/test/test_compile.py | 32 ++++++++++++++++++-------------- Python/instruction_sequence.c | 6 +++--- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 9f3ce388dd972e..484d72e63411a8 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -2429,24 +2429,28 @@ def compare_instructions(self, seq, expected): def test_basics(self): seq = _testinternalcapi.new_instruction_sequence() - seq.addop(opcode.opmap['LOAD_CONST'], 1, 1, 0, 0, 0) - seq.addop(opcode.opmap['JUMP'], lbl1 := seq.new_label(), 2, 0, 0, 0) - seq.addop(opcode.opmap['LOAD_CONST'], 1, 3, 0, 0, 0) - seq.addop(opcode.opmap['JUMP'], lbl2 := seq.new_label(), 4, 0, 0, 0) + + def add_op(seq, opname, oparg, bl, bc=0, el=0, ec=0): + seq.addop(opcode.opmap[opname], oparg, bl, bc, el, el) + + add_op(seq, 'LOAD_CONST', 1, 1) + add_op(seq, 'JUMP', lbl1 := seq.new_label(), 2) + add_op(seq, 'LOAD_CONST', 1, 3) + add_op(seq, 'JUMP', lbl2 := seq.new_label(), 4) seq.use_label(lbl1) - seq.addop(opcode.opmap['LOAD_CONST'], 2, 4, 0, 0, 0) + add_op(seq, 'LOAD_CONST', 2, 4) seq.use_label(lbl2) - seq.addop(opcode.opmap['RETURN_VALUE'], 0, 3, 0, 0, 0) - - expected = [('LOAD_CONST', 1, 1, 0, 0, 0), - ('JUMP', 4, 2, 0, 0, 0), - ('LOAD_CONST', 1, 3, 0, 0, 0), - ('JUMP', 5, 4, 0, 0, 0), - ('LOAD_CONST', 2, 4, 0, 0, 0), - ('RETURN_VALUE', None, 3, 0, 0, 0), + add_op(seq, 'RETURN_VALUE', 0, 3) + + expected = [('LOAD_CONST', 1, 1), + ('JUMP', 4, 2), + ('LOAD_CONST', 1, 3), + ('JUMP', 5, 4), + ('LOAD_CONST', 2, 4), + ('RETURN_VALUE', None, 3), ] - self.compare_instructions(seq, expected) + self.compare_instructions(seq, [ex + (0,0,0) for ex in expected]) def test_nested(self): seq = _testinternalcapi.new_instruction_sequence() diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index 44717b8cd2d88c..d843d5f16901b8 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -173,7 +173,7 @@ class InstructionSequenceType "_PyInstructionSequence *" "&_PyInstructionSequenc #include "clinic/instruction_sequence.c.h" static _PyInstructionSequence* -inst_seq_create_raw(void) +inst_seq_create(void) { _PyInstructionSequence *seq; seq = PyObject_GC_New(_PyInstructionSequence, &_PyInstructionSequence_Type); @@ -195,7 +195,7 @@ inst_seq_create_raw(void) PyObject* _PyInstructionSequence_New(void) { - _PyInstructionSequence *seq = inst_seq_create_raw(); + _PyInstructionSequence *seq = inst_seq_create(); if (seq == NULL) { return NULL; } @@ -213,7 +213,7 @@ static PyObject * inst_seq_new_impl(PyTypeObject *type) /*[clinic end generated code: output=98881de92c8876f6 input=b393150146849c74]*/ { - return (PyObject*)inst_seq_create_raw(); + return (PyObject*)inst_seq_create(); } /*[clinic input] From 0d5f90e8acb7662f6d7dfc207f4b451c53dbb944 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 17 Apr 2024 13:29:35 +0100 Subject: [PATCH 15/15] fix error handling in cfg_to_instruction_sequence --- Python/compile.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 1f4de1c2128ccf..3d856b7e4ddd97 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7601,13 +7601,18 @@ static PyObject * cfg_to_instruction_sequence(cfg_builder *g) { instr_sequence *seq = (instr_sequence *)_PyInstructionSequence_New(); - if (_PyCfg_ToInstructionSequence(g, seq) < 0) { - return NULL; - } - if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { - return NULL; + if (seq != NULL) { + if (_PyCfg_ToInstructionSequence(g, seq) < 0) { + goto error; + } + if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { + goto error; + } } return (PyObject*)seq; +error: + PyInstructionSequence_Fini(seq); + return NULL; } // C implementation of inspect.cleandoc()