GH-96793: Implement PEP 479 in bytecode. (GH-99006)

* Handle converting StopIteration to RuntimeError in bytecode.

* Add custom instruction for converting StopIteration into RuntimeError.

Author: markshannon

Doc/library/dis.rst

@@ -607,6 +607,15 @@ the original TOS1.
    .. versionadded:: 3.12
 
 
+.. opcode:: STOPITERATION_ERROR
+
+   Handles a StopIteration raised in a generator or coroutine.
+   If TOS is an instance of :exc:`StopIteration`, or :exc:`StopAsyncIteration`
+   replace it with a :exc:`RuntimeError`.
+
+   .. versionadded:: 3.12
+
+
 .. opcode:: BEFORE_ASYNC_WITH
 
    Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the

Include/internal/pycore_opcode.h

@@ -424,6 +424,7 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.12a1 3508 (Add CLEANUP_THROW)
 #     Python 3.12a1 3509 (Conditional jumps only jump forward)
 #     Python 3.12a1 3510 (FOR_ITER leaves iterator on the stack)
+#     Python 3.12a1 3511 (Add STOPITERATION_ERROR instruction)
 
 #     Python 3.13 will start with 3550
 
@@ -436,7 +437,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3510).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3511).to_bytes(2, 'little') + b'\r\n'
 
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 

Include/opcode.h

@@ -111,6 +111,8 @@ def pseudo_op(name, op, real_ops):
 def_op('STORE_SUBSCR', 60)
 def_op('DELETE_SUBSCR', 61)
 
+def_op('STOPITERATION_ERROR', 63)
+
 def_op('GET_ITER', 68)
 def_op('GET_YIELD_FROM_ITER', 69)
 def_op('PRINT_EXPR', 70)

Lib/importlib/_bootstrap_external.py

@@ -543,8 +543,10 @@ async def _asyncwith(c):
         >> COPY                     3
            POP_EXCEPT
            RERAISE                  1
+        >> STOPITERATION_ERROR
+           RERAISE                  1
 ExceptionTable:
-6 rows
+12 rows
 """ % (_asyncwith.__code__.co_firstlineno,
        _asyncwith.__code__.co_firstlineno + 1,
        _asyncwith.__code__.co_firstlineno + 2,

Lib/opcode.py

@@ -1439,7 +1439,7 @@ def bar(cls):
             check(bar, size('PP'))
         # generator
         def get_gen(): yield 1
-        check(get_gen(), size('P2P4P4c7P2ic??P'))
+        check(get_gen(), size('P2P4P4c7P2ic??2P'))
         # iterator
         check(iter('abc'), size('lP'))
         # callable-iterator

Lib/test/test_dis.py

@@ -346,7 +346,7 @@ def make_tracer():
         return Tracer()
 
     def compare_events(self, line_offset, events, expected_events):
-        events = [(l - line_offset, e) for (l, e) in events]
+        events = [(l - line_offset if l is not None else None, e) for (l, e) in events]
         if events != expected_events:
             self.fail(
                 "events did not match expectation:\n" +

Lib/test/test_sys.py

@@ -0,0 +1,2 @@
+Handle StopIteration and StopAsyncIteration raised in generator or
+coroutines in the bytecode, rather than in wrapping C code.

Lib/test/test_sys_settrace.py

@@ -246,25 +246,9 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
         }
     }
     else {
-        if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
-            const char *msg = "generator raised StopIteration";
-            if (PyCoro_CheckExact(gen)) {
-                msg = "coroutine raised StopIteration";
-            }
-            else if (PyAsyncGen_CheckExact(gen)) {
-                msg = "async generator raised StopIteration";
-            }
-            _PyErr_FormatFromCause(PyExc_RuntimeError, "%s", msg);
-        }
-        else if (PyAsyncGen_CheckExact(gen) &&
-                PyErr_ExceptionMatches(PyExc_StopAsyncIteration))
-        {
-            /* code in `gen` raised a StopAsyncIteration error:
-               raise a RuntimeError.
-            */
-            const char *msg = "async generator raised StopAsyncIteration";
-            _PyErr_FormatFromCause(PyExc_RuntimeError, "%s", msg);
-        }
+        assert(!PyErr_ExceptionMatches(PyExc_StopIteration));
+        assert(!PyAsyncGen_CheckExact(gen) ||
+            !PyErr_ExceptionMatches(PyExc_StopAsyncIteration));
     }
 
     /* generator can't be rerun, so release the frame */

Misc/NEWS.d/next/Core and Builtins/2022-11-02-14-42-35.gh-issue-96793.q0Oi74.rst

@@ -1085,6 +1085,49 @@ dummy_func(
             goto exception_unwind;
         }
 
+        inst(STOPITERATION_ERROR) {
+            assert(frame->owner == FRAME_OWNED_BY_GENERATOR);
+            PyObject *exc = TOP();
+            assert(PyExceptionInstance_Check(exc));
+            const char *msg = NULL;
+            if (PyErr_GivenExceptionMatches(exc, PyExc_StopIteration)) {
+                msg = "generator raised StopIteration";
+                if (frame->f_code->co_flags & CO_ASYNC_GENERATOR) {
+                    msg = "async generator raised StopIteration";
+                }
+                else if (frame->f_code->co_flags & CO_COROUTINE) {
+                    msg = "coroutine raised StopIteration";
+                }
+            }
+            else if ((frame->f_code->co_flags & CO_ASYNC_GENERATOR) &&
+                    PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration))
+            {
+                /* code in `gen` raised a StopAsyncIteration error:
+                raise a RuntimeError.
+                */
+                msg = "async generator raised StopAsyncIteration";
+            }
+            if (msg != NULL) {
+                PyObject *message = _PyUnicode_FromASCII(msg, strlen(msg));
+                if (message == NULL) {
+                    goto error;
+                }
+                PyObject *error = PyObject_CallOneArg(PyExc_RuntimeError, message);
+                if (error == NULL) {
+                    Py_DECREF(message);
+                    goto error;
+                }
+                assert(PyExceptionInstance_Check(error));
+                SET_TOP(error);
+                PyException_SetCause(error, exc);
+                Py_INCREF(exc);
+                PyException_SetContext(error, exc);
+                Py_DECREF(message);
+            }
+            DISPATCH();
+        }
+
+
         // stack effect: ( -- __0)
         inst(LOAD_ASSERTION_ERROR) {
             PyObject *value = PyExc_AssertionError;