Allow 0-prefixed underscore literals

Signed-off-by: Pablo Galindo <[email protected]>

Author: pablogsal

Lib/test/test_tokenize.py

@@ -284,7 +284,12 @@ def number_token(s):
                 # this won't work with compound complex inputs
                 continue
             self.assertEqual(number_token(lit), lit)
+        # Valid cases with extra underscores in the tokenize module
+        # See gh-105549 for context
+        extra_valid_cases = {"0_7", "09_99"}
         for lit in INVALID_UNDERSCORE_LITERALS:
+            if lit in extra_valid_cases:
+                continue
             try:
                 number_token(lit)
             except TokenError:
@@ -1873,6 +1878,33 @@ def test_indentation_semantics_retained(self):
         self.check_roundtrip(code)
 
 
+class InvalidPythonTests(TestCase):
+    def test_number_followed_by_name(self):
+        # See issue #gh-105549
+        source = "xsin(x)"
+        expected_tokens = [
+            TokenInfo(type=token.NAME, string='xsin', start=(1, 0), end=(1, 4), line='xsin(x)\n'),
+            TokenInfo(type=token.OP, string='(', start=(1, 4), end=(1, 5), line='xsin(x)\n'),
+            TokenInfo(type=token.NAME, string='x', start=(1, 5), end=(1, 6), line='xsin(x)\n'),
+            TokenInfo(type=token.OP, string=')', start=(1, 6), end=(1, 7), line='xsin(x)\n'),
+            TokenInfo(type=token.NEWLINE, string='', start=(1, 7), end=(1, 8), line='xsin(x)\n'),
+            TokenInfo(type=token.ENDMARKER, string='', start=(2, 0), end=(2, 0), line='')
+        ]
+
+        tokens = list(generate_tokens(StringIO(source).readline))
+        self.assertEqual(tokens, expected_tokens)
+
+    def test_number_starting_with_zero(self):
+        source = "01234"
+        expected_tokens = [
+            TokenInfo(type=token.NUMBER, string='01234', start=(1, 0), end=(1, 5), line='01234\n'),
+            TokenInfo(type=token.NEWLINE, string='', start=(1, 5), end=(1, 6), line='01234\n'),
+            TokenInfo(type=token.ENDMARKER, string='', start=(2, 0), end=(2, 0), line='')
+        ]
+
+        tokens = list(generate_tokens(StringIO(source).readline))
+        self.assertEqual(tokens, expected_tokens)
+
 class CTokenizeTest(TestCase):
     def check_tokenize(self, s, expected):
         # Format the tokens in s in a table format.

Parser/tokenizer.c

@@ -2325,7 +2325,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
                 else if (c == 'j' || c == 'J') {
                     goto imaginary;
                 }
-                else if (nonzero) {
+                else if (nonzero && !tok->tok_extra_tokens) {
                     /* Old-style octal: now disallowed. */
                     tok_backup(tok, c);
                     return MAKE_TOKEN(syntaxerror_known_range(