Skip to content

Commit 57f8f9a

Browse files
authored
gh-103718: Correctly set f-string buffers in all cases (GH-103815)
Turns out we always need to remember/restore fstring buffers in all of the stack of tokenizer modes, cause they might change to `TOK_REGULAR_MODE` and have newlines inside the braces (which is when we need to reallocate the buffer and restore the fstring ones).
1 parent 3df3b91 commit 57f8f9a

File tree

1 file changed

+6
-8
lines changed

1 file changed

+6
-8
lines changed

Parser/tokenizer.c

+6-8
Original file line numberDiff line numberDiff line change
@@ -371,10 +371,8 @@ remember_fstring_buffers(struct tok_state *tok)
371371

372372
for (index = tok->tok_mode_stack_index; index >= 0; --index) {
373373
mode = &(tok->tok_mode_stack[index]);
374-
if (mode->kind == TOK_FSTRING_MODE) {
375-
mode->f_string_start_offset = mode->f_string_start - tok->buf;
376-
mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf;
377-
}
374+
mode->f_string_start_offset = mode->f_string_start - tok->buf;
375+
mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf;
378376
}
379377
}
380378

@@ -387,10 +385,8 @@ restore_fstring_buffers(struct tok_state *tok)
387385

388386
for (index = tok->tok_mode_stack_index; index >= 0; --index) {
389387
mode = &(tok->tok_mode_stack[index]);
390-
if (mode->kind == TOK_FSTRING_MODE) {
391-
mode->f_string_start = tok->buf + mode->f_string_start_offset;
392-
mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset;
393-
}
388+
mode->f_string_start = tok->buf + mode->f_string_start_offset;
389+
mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset;
394390
}
395391
}
396392

@@ -1081,13 +1077,15 @@ tok_underflow_interactive(struct tok_state *tok) {
10811077
restore_fstring_buffers(tok);
10821078
}
10831079
else {
1080+
remember_fstring_buffers(tok);
10841081
ADVANCE_LINENO();
10851082
PyMem_Free(tok->buf);
10861083
tok->buf = newtok;
10871084
tok->cur = tok->buf;
10881085
tok->line_start = tok->buf;
10891086
tok->inp = strchr(tok->buf, '\0');
10901087
tok->end = tok->inp + 1;
1088+
restore_fstring_buffers(tok);
10911089
}
10921090
if (tok->done != E_OK) {
10931091
if (tok->prompt != NULL) {

0 commit comments

Comments
 (0)