@@ -148,23 +148,29 @@ def __init__(self, string: str, type: int) -> None:
148
148
COMMENT_CONTEXT = 2
149
149
150
150
151
- def lex (string : str , first_line : int = 1 ) -> List [Token ]:
151
+ def lex (string : str , first_line : int = 1 , pyversion : int = 3 ) -> List [Token ]:
152
152
"""Analyze string and return an array of token objects.
153
153
154
154
The last token is always Eof.
155
155
"""
156
- l = Lexer ()
156
+ l = Lexer (pyversion )
157
157
l .lex (string , first_line )
158
158
return l .tok
159
159
160
160
161
161
# Reserved words (not including operators)
162
- keywords = set ([
162
+ keywords_common = set ([
163
163
'as' , 'assert' , 'break' , 'class' , 'continue' , 'def' , 'del' , 'elif' ,
164
164
'else' , 'except' , 'finally' , 'from' , 'for' , 'global' , 'if' , 'import' ,
165
165
'lambda' , 'pass' , 'raise' , 'return' , 'try' , 'while' , 'with' ,
166
166
'yield' ])
167
167
168
+ # Reserved words specific for Python version 2
169
+ keywords2 = set (['print' ])
170
+
171
+ # Reserved words specific for Python version 3
172
+ keywords3 = set (['nonlocal' ])
173
+
168
174
# Alphabetical operators (reserved words)
169
175
alpha_operators = set (['in' , 'is' , 'not' , 'and' , 'or' ])
170
176
@@ -279,7 +285,7 @@ class Lexer:
279
285
# newlines within parentheses/brackets.
280
286
open_brackets = Undefined (List [str ])
281
287
282
- def __init__ (self ) -> None :
288
+ def __init__ (self , pyversion : int = 3 ) -> None :
283
289
self .map = [self .unknown_character ] * 256
284
290
self .tok = []
285
291
self .indents = [0 ]
@@ -302,6 +308,10 @@ def __init__(self) -> None:
302
308
('-+*/<>%&|^~=!,@' , self .lex_misc )]:
303
309
for c in seq :
304
310
self .map [ord (c )] = method
311
+ if pyversion == 2 :
312
+ self .keywords = keywords_common | keywords2
313
+ if pyversion == 3 :
314
+ self .keywords = keywords_common | keywords3
305
315
306
316
def lex (self , s : str , first_line : int ) -> None :
307
317
"""Lexically analyze a string, storing the tokens at the tok list."""
@@ -401,7 +411,7 @@ def lex_name(self) -> None:
401
411
Also deal with prefixed string literals such as r'...'.
402
412
"""
403
413
s = self .match (self .name_exp )
404
- if s in keywords :
414
+ if s in self . keywords :
405
415
self .add_token (Keyword (s ))
406
416
elif s in alpha_operators :
407
417
self .add_token (Op (s ))
0 commit comments