GH-131498: Another refactoring of the code generator (GH-131827)

* Rename 'defined' attribute to 'in_local' to more accurately reflect how it is used

* Make death of variables explicit even for array variables.

* Convert in_memory from boolean to stack offset

* Don't apply liveness analyis to optimizer generated code

* Add 'out' parameter to stack.pop

Author: markshannon

Lib/test/test_generated_cases.py

@@ -68,13 +68,15 @@ def test_effect_sizes(self):
             StackItem("b", None, "oparg*4"),
             StackItem("c", None, "1"),
         ]
-        stack.pop(z)
-        stack.pop(y)
-        stack.pop(x)
+        null = CWriter.null()
+        stack.pop(z, null)
+        stack.pop(y, null)
+        stack.pop(x, null)
         for out in outputs:
             stack.push(Local.undefined(out))
         self.assertEqual(stack.base_offset.to_c(), "-1 - oparg - oparg*2")
-        self.assertEqual(stack.top_offset.to_c(), "1 - oparg - oparg*2 + oparg*4")
+        self.assertEqual(stack.physical_sp.to_c(), "0")
+        self.assertEqual(stack.logical_sp.to_c(), "1 - oparg - oparg*2 + oparg*4")
 
 
 class TestGeneratedCases(unittest.TestCase):

Python/bytecodes.c

@@ -1484,7 +1484,7 @@ dummy_func(
             (void)counter;
         }
 
-        op(_UNPACK_SEQUENCE, (seq -- output[oparg], top[0])) {
+        op(_UNPACK_SEQUENCE, (seq -- unused[oparg], top[0])) {
             PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq);
             int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top);
             Py_DECREF(seq_o);
@@ -1533,7 +1533,7 @@ dummy_func(
             DECREF_INPUTS();
         }
 
-        inst(UNPACK_EX, (seq -- left[oparg & 0xFF], unused, right[oparg >> 8], top[0])) {
+        inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8], top[0])) {
             PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq);
             int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top);
             Py_DECREF(seq_o);

Python/generated_cases.c.h

@@ -1,7 +1,7 @@
 import contextlib
 from lexer import Token
 from typing import TextIO, Iterator
-
+from io import StringIO
 
 class CWriter:
     "A writer that understands tokens and how to format C code"
@@ -18,6 +18,10 @@ def __init__(self, out: TextIO, indent: int, line_directives: bool):
         self.pending_spill = False
         self.pending_reload = False
 
+    @staticmethod
+    def null() -> "CWriter":
+        return CWriter(StringIO(), 0, False)
+
     def set_position(self, tkn: Token) -> None:
         if self.last_token is not None:
             if self.last_token.end_line < tkn.line:

Tools/cases_generator/cwriter.py

@@ -205,9 +205,9 @@ def error_if(
         next(tkn_iter)  # Semi colon
         storage.clear_inputs("at ERROR_IF")
 
-        c_offset = storage.stack.peek_offset()
+        c_offset = storage.stack.sp_offset()
         try:
-            offset = -int(c_offset)
+            offset = int(c_offset)
         except ValueError:
             offset = -1
         self.out.emit(self.goto_error(offset, label, storage))

Tools/cases_generator/generators_common.py

@@ -100,7 +100,7 @@ def generate_stack_effect_functions(analysis: Analysis, out: CWriter) -> None:
     def add(inst: Instruction | PseudoInstruction) -> None:
         stack = get_stack_effect(inst)
         popped = (-stack.base_offset).to_c()
-        pushed = (stack.top_offset - stack.base_offset).to_c()
+        pushed = (stack.logical_sp - stack.base_offset).to_c()
         popped_data.append((inst.name, popped))
         pushed_data.append((inst.name, pushed))
 

Tools/cases_generator/opcode_metadata_generator.py

@@ -72,14 +72,15 @@ def decref_inputs(
 
 
 def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None:
+    null = CWriter.null()
     for var in reversed(uop.stack.inputs):
-        stack.pop(var)
-    top_offset = stack.top_offset.copy()
+        stack.pop(var, null)
+    offset = stack.base_offset - stack.physical_sp
     for var in uop.stack.outputs:
         if var.is_array() and not var.peek and not var.name == "unused":
-            c_offset = top_offset.to_c()
+            c_offset = offset.to_c()
             out.emit(f"{var.name} = &stack_pointer[{c_offset}];\n")
-        top_offset.push(var)
+        offset = offset.push(var)
     for var in uop.stack.outputs:
         local = Local.undefined(var)
         stack.push(local)
@@ -123,9 +124,7 @@ def write_uop(
     try:
         out.start_line()
         if override:
-            code_list, storage = Storage.for_uop(stack, prototype, check_liveness=False)
-            for code in code_list:
-                out.emit(code)
+            storage = Storage.for_uop(stack, prototype, out, check_liveness=False)
         if debug:
             args = []
             for input in prototype.stack.inputs: