Add macro() and op() DSL features

Author: gvanrossum

Python/bytecodes.c

@@ -70,6 +70,8 @@ do { \
 #define DISPATCH_SAME_OPARG() ((void)0)
 
 #define inst(name, ...) case name:
+#define op(name, ...) /* NAME is ignored */
+#define macro(name) static int MACRO_##name
 #define super(name) static int SUPER_##name
 #define family(name, ...) static int family_##name
 
@@ -156,10 +158,7 @@ dummy_func(
             res = NULL;
         }
 
-        inst(END_FOR, (value1, value2 --)) {
-            Py_DECREF(value1);
-            Py_DECREF(value2);
-        }
+        macro(END_FOR) = POP_TOP + POP_TOP;
 
         inst(UNARY_POSITIVE, (value -- res)) {
             res = PyNumber_Positive(value);

Python/generated_cases.c.h

@@ -179,8 +179,8 @@ def __init__(self, filename: str):
         with open(filename) as f:
             self.src = f.read()
 
-    instrs: dict[str, Instruction]
-    supers: dict[str, parser.Super]
+    instrs: dict[str, Instruction]  # Includes ops
+    supers: dict[str, parser.Super]  # Includes macros
     families: dict[str, parser.Family]
 
     def parse(self) -> None:
@@ -303,6 +303,8 @@ def write_instructions(self, filename: str) -> None:
 
             # Write regular instructions
             for name, instr in self.instrs.items():
+                if instr.kind != "inst":
+                    continue # ops are not real instructions
                 f.write(f"\n{indent}TARGET({name}) {{\n")
                 if instr.predicted:
                     f.write(f"{indent}    PREDICTED({name});\n")
@@ -316,7 +318,7 @@ def write_instructions(self, filename: str) -> None:
                 components = [self.instrs[name] for name in sup.ops]
                 f.write(f"\n{indent}TARGET({sup.name}) {{\n")
                 for i, instr in enumerate(components):
-                    if i > 0:
+                    if i > 0 and sup.kind == "super":
                         f.write(f"{indent}    NEXTOPARG();\n")
                         f.write(f"{indent}    next_instr++;\n")
                     f.write(f"{indent}    {{\n")

Tools/cases_generator/generate_cases.py

@@ -1,7 +1,7 @@
 """Parser for bytecodes.inst."""
 
 from dataclasses import dataclass, field
-from typing import NamedTuple, Callable, TypeVar
+from typing import NamedTuple, Callable, TypeVar, Literal
 
 import lexer as lx
 from plexer import PLexer
@@ -74,16 +74,22 @@ class CacheEffect(Node):
 
 @dataclass
 class InstHeader(Node):
+    kind: Literal["inst", "op"]
     name: str
     inputs: list[InputEffect]
     outputs: list[OutputEffect]
 
 
 @dataclass
 class InstDef(Node):
+    # TODO: Merge InstHeader and InstDef
     header: InstHeader
     block: Block
 
+    @property
+    def kind(self) -> str:
+        return self.header.kind
+
     @property
     def name(self) -> str:
         return self.header.name
@@ -99,6 +105,7 @@ def outputs(self) -> list[StackEffect]:
 
 @dataclass
 class Super(Node):
+    kind: Literal["macro", "super"]
     name: str
     ops: list[str]
 
@@ -122,10 +129,12 @@ def inst_def(self) -> InstDef | None:
 
     @contextual
     def inst_header(self) -> InstHeader | None:
-        # inst(NAME) | inst(NAME, (inputs -- outputs))
+        # inst(NAME)
+        #   | inst(NAME, (inputs -- outputs))
+        #   | op(NAME, (inputs -- outputs))
         # TODO: Error out when there is something unexpected.
         # TODO: Make INST a keyword in the lexer.
-        if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "inst":
+        if (tkn := self.expect(lx.IDENTIFIER)) and (kind := tkn.text) in ("inst", "op"):
             if (self.expect(lx.LPAREN)
                     and (tkn := self.expect(lx.IDENTIFIER))):
                 name = tkn.text
@@ -134,9 +143,10 @@ def inst_header(self) -> InstHeader | None:
                     if self.expect(lx.RPAREN):
                         if ((tkn := self.peek())
                                 and tkn.kind == lx.LBRACE):
-                            return InstHeader(name, inp, outp)
-                elif self.expect(lx.RPAREN):
-                    return InstHeader(name, [], [])
+                            return InstHeader(kind, name, inp, outp)
+                elif self.expect(lx.RPAREN) and kind == "inst":
+                    # No legacy stack effect if kind is "op".
+                    return InstHeader(kind, name, [], [])
         return None
 
     def stack_effect(self) -> tuple[list[InputEffect], list[OutputEffect]]:
@@ -200,13 +210,13 @@ def output(self) -> OutputEffect | None:
 
     @contextual
     def super_def(self) -> Super | None:
-        if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "super":
+        if (tkn := self.expect(lx.IDENTIFIER)) and (kind := tkn.text) in ("super", "macro"):
             if self.expect(lx.LPAREN):
                 if (tkn := self.expect(lx.IDENTIFIER)):
                     if self.expect(lx.RPAREN):
                         if self.expect(lx.EQUALS):
                             if ops := self.ops():
-                                res = Super(tkn.text, ops)
+                                res = Super(kind, tkn.text, ops)
                                 return res
 
     def ops(self) -> list[str] | None:
@@ -278,7 +288,7 @@ def c_blob(self) -> list[lx.Token]:
         filename = sys.argv[1]
         if filename == "-c" and sys.argv[2:]:
             src = sys.argv[2]
-            filename = None
+            filename = "<string>"
         else:
             with open(filename) as f:
                 src = f.read()
@@ -287,7 +297,7 @@ def c_blob(self) -> list[lx.Token]:
             end = srclines.index("// END BYTECODES //")
             src = "\n".join(srclines[begin+1 : end])
     else:
-        filename = None
+        filename = "<default>"
         src = "if (x) { x.foo; // comment\n}"
     parser = Parser(src, filename)
     x = parser.inst_def() or parser.super_def() or parser.family_def()