Skip to content

Commit 93ee708

Browse files
committed
gh-105549: Tokenize separately NUMBER and NAME tokens
1 parent d7f46bc commit 93ee708

File tree

2 files changed

+11
-2
lines changed

2 files changed

+11
-2
lines changed
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Tokenize separately `NUMBER` and `NAME` tokens that are not ambiguous. Patch
2+
by Pablo Galindo

Parser/tokenizer.c

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1600,8 +1600,12 @@ lookahead(struct tok_state *tok, const char *test)
16001600
}
16011601

16021602
static int
1603-
verify_end_of_number(struct tok_state *tok, int c, const char *kind)
1604-
{
1603+
verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
1604+
if (tok->tok_extra_tokens) {
1605+
// When we are parsing extra tokens, we don't want to emit warnings
1606+
// about invalid literals, because we want to be a bit more liberal.
1607+
return 1;
1608+
}
16051609
/* Emit a deprecation warning only if the numeric literal is immediately
16061610
* followed by one of keywords which can occur after a numeric literal
16071611
* in valid code: "and", "else", "for", "if", "in", "is" and "or".
@@ -1659,6 +1663,9 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind)
16591663
static int
16601664
verify_identifier(struct tok_state *tok)
16611665
{
1666+
if (tok->tok_extra_tokens) {
1667+
return 1;
1668+
}
16621669
PyObject *s;
16631670
if (tok->decoding_erred)
16641671
return 0;

0 commit comments

Comments
 (0)