Skip to content

Commit b41a3b8

Browse files
authored
Merge pull request python#51 from isidentical/more-simplifications
2 parents f1246db + bd7483e commit b41a3b8

File tree

5 files changed

+15
-6
lines changed

5 files changed

+15
-6
lines changed

Include/internal/pycore_token.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ extern "C" {
9090
(x) == NEWLINE || \
9191
(x) == INDENT || \
9292
(x) == DEDENT)
93+
#define ISSTRINGLIT(x) ((x) == STRING || \
94+
(x) == FSTRING_MIDDLE || \
95+
(x) == FSTRING_END)
9396

9497

9598
// Symbols exported for test_peg_generator

Parser/tokenizer.c

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1563,7 +1563,12 @@ token_setup(struct tok_state *tok, struct token *token, int type, const char *st
15631563
{
15641564
assert((start == NULL && end == NULL) || (start != NULL && end != NULL));
15651565
token->level = tok->level;
1566-
token->lineno = type == STRING ? tok->first_lineno : (type == FSTRING_MIDDLE || type == FSTRING_END ? tok->fstring_first_constant_lineno : tok->lineno);
1566+
if (ISSTRINGLIT(type)) {
1567+
token->lineno = tok->first_lineno;
1568+
}
1569+
else {
1570+
token->lineno = tok->lineno;
1571+
}
15671572
token->end_lineno = tok->lineno;
15681573
token->col_offset = token->end_col_offset = -1;
15691574
token->start = start;
@@ -2423,7 +2428,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
24232428
const char *p_start = NULL;
24242429
const char *p_end = NULL;
24252430
tok->start = tok->cur;
2426-
tok->fstring_first_constant_lineno = tok->lineno;
2431+
tok->first_lineno = tok->lineno;
24272432
tok->starting_col_offset = tok->col_offset;
24282433

24292434
// If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize

Parser/tokenizer.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,6 @@ struct tok_state {
8282
int lineno; /* Current line number */
8383
int first_lineno; /* First line of a single line or multi line string
8484
expression (cf. issue 16806) */
85-
int fstring_first_constant_lineno; /* First line number of a single line or multiline
86-
constant part of an f-string*/
8785
int starting_col_offset; /* The column offset at the beginning of a token */
8886
int col_offset; /* Current col offset */
8987
int level; /* () [] {} Parentheses nesting level */

Python/Python-tokenize.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,8 @@ tokenizeriter_next(tokenizeriterobject *it)
8686
Py_DECREF(str);
8787
return NULL;
8888
}
89-
const char *line_start = type == STRING ? it->tok->multi_line_start : it->tok->line_start;
90-
int lineno = type == STRING ? it->tok->first_lineno : it->tok->lineno;
89+
const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
90+
int lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
9191
int end_lineno = it->tok->lineno;
9292
int col_offset = -1;
9393
int end_col_offset = -1;

Tools/build/generate_token.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ def update_file(file, content):
8080
(x) == NEWLINE || \\
8181
(x) == INDENT || \\
8282
(x) == DEDENT)
83+
#define ISSTRINGLIT(x) ((x) == STRING || \\
84+
(x) == FSTRING_MIDDLE || \\
85+
(x) == FSTRING_END)
8386
8487
8588
// Symbols exported for test_peg_generator

0 commit comments

Comments
 (0)