@@ -284,7 +284,12 @@ def number_token(s):
284
284
# this won't work with compound complex inputs
285
285
continue
286
286
self .assertEqual (number_token (lit ), lit )
287
+ # Valid cases with extra underscores in the tokenize module
288
+ # See gh-105549 for context
289
+ extra_valid_cases = {"0_7" , "09_99" }
287
290
for lit in INVALID_UNDERSCORE_LITERALS :
291
+ if lit in extra_valid_cases :
292
+ continue
288
293
try :
289
294
number_token (lit )
290
295
except TokenError :
@@ -1873,6 +1878,34 @@ def test_indentation_semantics_retained(self):
1873
1878
self .check_roundtrip (code )
1874
1879
1875
1880
1881
+ class InvalidPythonTests (TestCase ):
1882
+ def test_number_followed_by_name (self ):
1883
+ # See issue #gh-105549
1884
+ source = "2sin(x)"
1885
+ expected_tokens = [
1886
+ TokenInfo (type = token .NUMBER , string = '2' , start = (1 , 0 ), end = (1 , 1 ), line = '2sin(x)' ),
1887
+ TokenInfo (type = token .NAME , string = 'sin' , start = (1 , 1 ), end = (1 , 4 ), line = '2sin(x)' ),
1888
+ TokenInfo (type = token .OP , string = '(' , start = (1 , 4 ), end = (1 , 5 ), line = '2sin(x)' ),
1889
+ TokenInfo (type = token .NAME , string = 'x' , start = (1 , 5 ), end = (1 , 6 ), line = '2sin(x)' ),
1890
+ TokenInfo (type = token .OP , string = ')' , start = (1 , 6 ), end = (1 , 7 ), line = '2sin(x)' ),
1891
+ TokenInfo (type = token .NEWLINE , string = '' , start = (1 , 7 ), end = (1 , 8 ), line = '2sin(x)' ),
1892
+ TokenInfo (type = token .ENDMARKER , string = '' , start = (2 , 0 ), end = (2 , 0 ), line = '' )
1893
+ ]
1894
+
1895
+ tokens = list (generate_tokens (StringIO (source ).readline ))
1896
+ self .assertEqual (tokens , expected_tokens )
1897
+
1898
+ def test_number_starting_with_zero (self ):
1899
+ source = "01234"
1900
+ expected_tokens = [
1901
+ TokenInfo (type = token .NUMBER , string = '01234' , start = (1 , 0 ), end = (1 , 5 ), line = '01234' ),
1902
+ TokenInfo (type = token .NEWLINE , string = '' , start = (1 , 5 ), end = (1 , 6 ), line = '01234' ),
1903
+ TokenInfo (type = token .ENDMARKER , string = '' , start = (2 , 0 ), end = (2 , 0 ), line = '' )
1904
+ ]
1905
+
1906
+ tokens = list (generate_tokens (StringIO (source ).readline ))
1907
+ self .assertEqual (tokens , expected_tokens )
1908
+
1876
1909
class CTokenizeTest (TestCase ):
1877
1910
def check_tokenize (self , s , expected ):
1878
1911
# Format the tokens in s in a table format.
0 commit comments