Skip to content

Commit 0b50a4f

Browse files
markshannonmarkshannon
authored
bpo-46039: Split yield from in two (GH-30035)
* Split YIELD_FROM opcode into SEND and JUMP_ABSOLUTE. * Remove YIELD_FROM opcode.
1 parent 86de995 commit 0b50a4f

File tree

10 files changed

+91
-53
lines changed

10 files changed

+91
-53
lines changed

‎Include/internal/pycore_frame.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ static inline PyObject **_PyFrame_Stackbase(InterpreterFrame *f) {
6666

6767
static inline PyObject *_PyFrame_StackPeek(InterpreterFrame *f) {
6868
assert(f->stacktop > f->f_code->co_nlocalsplus);
69+
assert(f->localsplus[f->stacktop-1] != NULL);
6970
return f->localsplus[f->stacktop-1];
7071
}
7172

‎Include/opcode.h

Lines changed: 12 additions & 12 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/importlib/_bootstrap_external.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,7 @@ def _write_atomic(path, data, mode=0o666):
373373
# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode)
374374
# Python 3.11a3 3466 (bpo-45292: PEP-654 except*)
375375
# Python 3.11a4 3467 (Change CALL_xxx opcodes)
376+
# Python 3.11a4 3468 (Add SEND opcode)
376377

377378
#
378379
# MAGIC must change whenever the bytecode emitted by the compiler may no
@@ -382,7 +383,7 @@ def _write_atomic(path, data, mode=0o666):
382383
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
383384
# in PC/launcher.c must also be updated.
384385

385-
MAGIC_NUMBER = (3467).to_bytes(2, 'little') + b'\r\n'
386+
MAGIC_NUMBER = (3468).to_bytes(2, 'little') + b'\r\n'
386387
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
387388

388389
_PYCACHE = '__pycache__'

‎Lib/opcode.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ def jabs_op(name, op):
9393
def_op('GET_YIELD_FROM_ITER', 69)
9494
def_op('PRINT_EXPR', 70)
9595
def_op('LOAD_BUILD_CLASS', 71)
96-
def_op('YIELD_FROM', 72)
96+
9797
def_op('GET_AWAITABLE', 73)
9898
def_op('LOAD_ASSERTION_ERROR', 74)
9999

@@ -143,7 +143,7 @@ def jabs_op(name, op):
143143
def_op('COPY', 120)
144144
jabs_op('JUMP_IF_NOT_EXC_MATCH', 121)
145145
def_op('BINARY_OP', 122)
146-
146+
jrel_op('SEND', 123) # Number of bytes to skip
147147
def_op('LOAD_FAST', 124) # Local variable number
148148
haslocal.append(124)
149149
def_op('STORE_FAST', 125) # Local variable number

‎Misc/NEWS.d/next/Core and Builtins/2021-12-13-17-01-13.bpo-46039.TrCBbF.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Remove the ``YIELD_FROM`` instruction and replace it with the ``SEND``
2+
instruction which performs the same operation, but without the loop.

‎Objects/frameobject.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,13 @@ mark_stacks(PyCodeObject *code_obj, int len)
249249
next_stack = pop_value(pop_value(pop_value(next_stack)));
250250
stacks[i+1] = next_stack;
251251
break;
252-
252+
case SEND:
253+
j = get_arg(code, i) + i + 1;
254+
assert(j < len);
255+
assert(stacks[j] == UNINITIALIZED || stacks[j] == pop_value(next_stack));
256+
stacks[j] = pop_value(next_stack);
257+
stacks[i+1] = next_stack;
258+
break;
253259
case JUMP_FORWARD:
254260
j = get_arg(code, i) + i + 1;
255261
assert(j < len);

‎Objects/genobject.c

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
#include "pycore_frame.h" // InterpreterFrame
1111
#include "frameobject.h" // PyFrameObject
1212
#include "structmember.h" // PyMemberDef
13-
#include "opcode.h" // YIELD_FROM
13+
#include "opcode.h" // SEND
1414

1515
static PyObject *gen_close(PyGenObject *, PyObject *);
1616
static PyObject *async_gen_asend_new(PyAsyncGenObject *, PyObject *);
@@ -356,14 +356,14 @@ _PyGen_yf(PyGenObject *gen)
356356
unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
357357

358358
if (frame->f_lasti < 0) {
359-
/* Return immediately if the frame didn't start yet. YIELD_FROM
359+
/* Return immediately if the frame didn't start yet. SEND
360360
always come after LOAD_CONST: a code object should not start
361-
with YIELD_FROM */
362-
assert(code[0] != YIELD_FROM);
361+
with SEND */
362+
assert(code[0] != SEND);
363363
return NULL;
364364
}
365365

366-
if (code[(frame->f_lasti+1)*sizeof(_Py_CODEUNIT)] != YIELD_FROM)
366+
if (code[frame->f_lasti*sizeof(_Py_CODEUNIT)] != SEND || frame->stacktop < 0)
367367
return NULL;
368368
yf = _PyFrame_StackPeek(frame);
369369
Py_INCREF(yf);
@@ -486,9 +486,13 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
486486
ret = _PyFrame_StackPop((InterpreterFrame *)gen->gi_iframe);
487487
assert(ret == yf);
488488
Py_DECREF(ret);
489-
/* Termination repetition of YIELD_FROM */
489+
/* Termination repetition of SEND loop */
490490
assert(frame->f_lasti >= 0);
491-
frame->f_lasti += 1;
491+
PyObject *bytecode = gen->gi_code->co_code;
492+
unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
493+
assert(code[frame->f_lasti*sizeof(_Py_CODEUNIT)] == SEND);
494+
int jump = code[frame->f_lasti*sizeof(_Py_CODEUNIT)+1];
495+
frame->f_lasti += jump;
492496
if (_PyGen_FetchStopIterationValue(&val) == 0) {
493497
ret = gen_send(gen, val);
494498
Py_DECREF(val);

‎Python/ceval.c

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1798,7 +1798,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
17981798
if (_Py_atomic_load_relaxed(eval_breaker)) {
17991799
opcode = _Py_OPCODE(*next_instr);
18001800
if (opcode != BEFORE_ASYNC_WITH &&
1801-
opcode != YIELD_FROM) {
1801+
opcode != SEND &&
1802+
_Py_OPCODE(next_instr[-1]) != SEND) {
18021803
/* Few cases where we skip running signal handlers and other
18031804
pending calls:
18041805
- If we're about to enter the 'with:'. It will prevent
@@ -2642,8 +2643,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
26422643
DISPATCH();
26432644
}
26442645

2645-
TARGET(YIELD_FROM) {
2646+
TARGET(SEND) {
26462647
assert(frame->depth == 0);
2648+
assert(STACK_LEVEL() >= 2);
26472649
PyObject *v = POP();
26482650
PyObject *receiver = TOP();
26492651
PySendResult gen_status;
@@ -2680,17 +2682,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
26802682
}
26812683
if (gen_status == PYGEN_RETURN) {
26822684
assert (retval != NULL);
2683-
26842685
Py_DECREF(receiver);
26852686
SET_TOP(retval);
2686-
retval = NULL;
2687+
JUMPBY(oparg);
26872688
DISPATCH();
26882689
}
26892690
assert (gen_status == PYGEN_NEXT);
2690-
/* receiver remains on stack, retval is value to be yielded */
2691-
/* and repeat... */
2692-
assert(frame->f_lasti > 0);
2693-
frame->f_lasti -= 1;
2691+
assert (retval != NULL);
26942692
frame->f_state = FRAME_SUSPENDED;
26952693
_PyFrame_SetStackPointer(frame, stack_pointer);
26962694
TRACE_FUNCTION_EXIT();
@@ -6770,8 +6768,11 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
67706768
return -1;
67716769
}
67726770
if (line != -1 && f->f_trace_lines) {
6773-
/* Trace backward edges or if line number has changed */
6774-
if (frame->f_lasti < instr_prev || line != lastline) {
6771+
/* Trace backward edges (except in 'yield from') or if line number has changed */
6772+
int trace = line != lastline ||
6773+
(frame->f_lasti < instr_prev &&
6774+
_Py_OPCODE(frame->f_code->co_firstinstr[frame->f_lasti]) != SEND);
6775+
if (trace) {
67756776
result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
67766777
}
67776778
}

‎Python/compile.c

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1045,8 +1045,6 @@ stack_effect(int opcode, int oparg, int jump)
10451045
return 0;
10461046
case YIELD_VALUE:
10471047
return 0;
1048-
case YIELD_FROM:
1049-
return -1;
10501048
case POP_BLOCK:
10511049
return 0;
10521050
case POP_EXCEPT:
@@ -1065,7 +1063,8 @@ stack_effect(int opcode, int oparg, int jump)
10651063
case FOR_ITER:
10661064
/* -1 at end of iterator, 1 if continue iterating. */
10671065
return jump > 0 ? -1 : 1;
1068-
1066+
case SEND:
1067+
return jump > 0 ? -1 : 0;
10691068
case STORE_ATTR:
10701069
return -2;
10711070
case DELETE_ATTR:
@@ -1667,6 +1666,9 @@ compiler_addop_j_noline(struct compiler *c, int opcode, basicblock *b)
16671666
the ASDL name to synthesize the name of the C type and the visit function.
16681667
*/
16691668

1669+
#define ADD_YIELD_FROM(C) \
1670+
RETURN_IF_FALSE(compiler_add_yield_from((C)))
1671+
16701672
#define VISIT(C, TYPE, V) {\
16711673
if (!compiler_visit_ ## TYPE((C), (V))) \
16721674
return 0; \
@@ -1819,6 +1821,24 @@ compiler_call_exit_with_nones(struct compiler *c) {
18191821
return 1;
18201822
}
18211823

1824+
static int
1825+
compiler_add_yield_from(struct compiler *c)
1826+
{
1827+
basicblock *start, *jump, *exit;
1828+
start = compiler_new_block(c);
1829+
jump = compiler_new_block(c);
1830+
exit = compiler_new_block(c);
1831+
if (start == NULL || jump == NULL || exit == NULL) {
1832+
return 0;
1833+
}
1834+
compiler_use_next_block(c, start);
1835+
ADDOP_JUMP(c, SEND, exit);
1836+
compiler_use_next_block(c, jump);
1837+
ADDOP_JUMP(c, JUMP_ABSOLUTE, start);
1838+
compiler_use_next_block(c, exit);
1839+
return 1;
1840+
}
1841+
18221842
/* Unwind a frame block. If preserve_tos is true, the TOS before
18231843
* popping the blocks will be restored afterwards, unless another
18241844
* return, break or continue is found. In which case, the TOS will
@@ -1893,7 +1913,7 @@ compiler_unwind_fblock(struct compiler *c, struct fblockinfo *info,
18931913
if (info->fb_type == ASYNC_WITH) {
18941914
ADDOP(c, GET_AWAITABLE);
18951915
ADDOP_LOAD_CONST(c, Py_None);
1896-
ADDOP(c, YIELD_FROM);
1916+
ADD_YIELD_FROM(c);
18971917
}
18981918
ADDOP(c, POP_TOP);
18991919
/* The exit block should appear to execute after the
@@ -3006,7 +3026,7 @@ compiler_async_for(struct compiler *c, stmt_ty s)
30063026
ADDOP_JUMP(c, SETUP_FINALLY, except);
30073027
ADDOP(c, GET_ANEXT);
30083028
ADDOP_LOAD_CONST(c, Py_None);
3009-
ADDOP(c, YIELD_FROM);
3029+
ADD_YIELD_FROM(c);
30103030
ADDOP(c, POP_BLOCK); /* for SETUP_FINALLY */
30113031

30123032
/* Success block for __anext__ */
@@ -5192,7 +5212,7 @@ compiler_async_comprehension_generator(struct compiler *c,
51925212
ADDOP_JUMP(c, SETUP_FINALLY, except);
51935213
ADDOP(c, GET_ANEXT);
51945214
ADDOP_LOAD_CONST(c, Py_None);
5195-
ADDOP(c, YIELD_FROM);
5215+
ADD_YIELD_FROM(c);
51965216
ADDOP(c, POP_BLOCK);
51975217
VISIT(c, expr, gen->target);
51985218

@@ -5342,7 +5362,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
53425362
if (is_async_generator && type != COMP_GENEXP) {
53435363
ADDOP(c, GET_AWAITABLE);
53445364
ADDOP_LOAD_CONST(c, Py_None);
5345-
ADDOP(c, YIELD_FROM);
5365+
ADD_YIELD_FROM(c);
53465366
}
53475367

53485368
return 1;
@@ -5493,7 +5513,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
54935513
ADDOP(c, BEFORE_ASYNC_WITH);
54945514
ADDOP(c, GET_AWAITABLE);
54955515
ADDOP_LOAD_CONST(c, Py_None);
5496-
ADDOP(c, YIELD_FROM);
5516+
ADD_YIELD_FROM(c);
54975517

54985518
ADDOP_JUMP(c, SETUP_WITH, final);
54995519

@@ -5530,7 +5550,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
55305550
return 0;
55315551
ADDOP(c, GET_AWAITABLE);
55325552
ADDOP_LOAD_CONST(c, Py_None);
5533-
ADDOP(c, YIELD_FROM);
5553+
ADD_YIELD_FROM(c);
55345554

55355555
ADDOP(c, POP_TOP);
55365556

@@ -5544,7 +5564,7 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
55445564
ADDOP(c, WITH_EXCEPT_START);
55455565
ADDOP(c, GET_AWAITABLE);
55465566
ADDOP_LOAD_CONST(c, Py_None);
5547-
ADDOP(c, YIELD_FROM);
5567+
ADD_YIELD_FROM(c);
55485568
compiler_with_except_finish(c, cleanup);
55495569

55505570
compiler_use_next_block(c, exit);
@@ -5701,7 +5721,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
57015721
VISIT(c, expr, e->v.YieldFrom.value);
57025722
ADDOP(c, GET_YIELD_FROM_ITER);
57035723
ADDOP_LOAD_CONST(c, Py_None);
5704-
ADDOP(c, YIELD_FROM);
5724+
ADD_YIELD_FROM(c);
57055725
break;
57065726
case Await_kind:
57075727
if (!IS_TOP_LEVEL_AWAIT(c)){
@@ -5718,7 +5738,7 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
57185738
VISIT(c, expr, e->v.Await.value);
57195739
ADDOP(c, GET_AWAITABLE);
57205740
ADDOP_LOAD_CONST(c, Py_None);
5721-
ADDOP(c, YIELD_FROM);
5741+
ADD_YIELD_FROM(c);
57225742
break;
57235743
case Compare_kind:
57245744
return compiler_compare(c, e);
@@ -7544,10 +7564,13 @@ normalize_jumps(struct assembler *a)
75447564
continue;
75457565
}
75467566
struct instr *last = &b->b_instr[b->b_iused-1];
7547-
if (last->i_opcode == JUMP_ABSOLUTE &&
7548-
last->i_target->b_visited == 0
7549-
) {
7550-
last->i_opcode = JUMP_FORWARD;
7567+
if (last->i_opcode == JUMP_ABSOLUTE) {
7568+
if (last->i_target->b_visited == 0) {
7569+
last->i_opcode = JUMP_FORWARD;
7570+
}
7571+
else if (b->b_iused >= 2 && b->b_instr[b->b_iused-2].i_opcode == SEND) {
7572+
last->i_opcode = JUMP_ABSOLUTE_QUICK;
7573+
}
75517574
}
75527575
}
75537576
}

0 commit comments

Comments
 (0)