Simplified lexing of ellipsis

Author: spkersten

mypy/lex.py

@@ -117,6 +117,10 @@ class Colon(Token):
     pass
 
 
+class EllipsisToken(Token):
+    pass
+
+
 class Op(Token):
     """Operator (e.g. '+' or 'in')"""
 
@@ -146,7 +150,6 @@ def __init__(self, string: str, type: int) -> None:
 INVALID_UTF8_SEQUENCE = 5
 INVALID_BACKSLASH = 6
 INVALID_DEDENT = 7
-ELLIPSIS_ERROR = 8
 
 # Encoding contexts
 STR_CONTEXT = 1
@@ -176,7 +179,7 @@ def lex(string: str, first_line: int = 1, pyversion: int = 3) -> List[Token]:
 keywords2 = set([]) # type: Set[str]
 
 # Reserved words specific for Python version 3
-keywords3 = set(['nonlocal', '...'])
+keywords3 = set(['nonlocal'])
 
 # Alphabetical operators (reserved words)
 alpha_operators = set(['in', 'is', 'not', 'and', 'or'])
@@ -429,11 +432,7 @@ def lex_number(self) -> None:
             self.add_token(ComplexLit(sc))
 
     def lex_ellipsis(self) -> None:
-        ellipsis = self.match(re.compile('\.\.\.[0-9a-zA-Z_]*'))
-        if len(ellipsis) == 3:
-            self.add_token(Keyword('...'))
-        else:
-            self.add_token(LexError(ellipsis, ELLIPSIS_ERROR))
+        self.add_token(EllipsisToken('...'))
 
     name_exp = re.compile('[a-zA-Z_][a-zA-Z0-9_]*')
 

mypy/parse.py

@@ -11,7 +11,8 @@
 from mypy import lex
 from mypy.lex import (
     Token, Eof, Bom, Break, Name, Colon, Dedent, IntLit, StrLit, BytesLit,
-    UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit
+    UnicodeLit, FloatLit, Op, Indent, Keyword, Punct, LexError, ComplexLit,
+    EllipsisToken
 )
 import mypy.types
 from mypy.nodes import (
@@ -1128,7 +1129,7 @@ def parse_expression(self, prec: int = 0, star_expr_allowed: bool = False) -> No
                 expr = self.parse_complex_expr()
             elif isinstance(t, Keyword) and s == "yield":
                 expr = self.parse_yield_from_expr() # The expression yield from and yield to assign
-            elif isinstance(t, Keyword) and s == '...':
+            elif isinstance(t, EllipsisToken):
                 expr = self.parse_ellipsis()
             else:
                 # Invalid expression.
@@ -1856,8 +1857,6 @@ def token_repr(tok: Token) -> str:
                 return 'non-ASCII character in string'
             elif t == lex.INVALID_DEDENT:
                 return 'inconsistent indentation'
-            elif t == lex.ELLIPSIS_ERROR:
-                return 'invalid ellipsis'
         raise ValueError('Unknown token {}'.format(repr(tok)))
 
 

mypy/test/data/parse-errors.test

@@ -380,6 +380,6 @@ file, line 1: Parse error before end of line
 ..._
 ...a
 [out]
-file, line 1: Invalid ellipsis
-file, line 2: Invalid ellipsis
-file, line 3: Invalid ellipsis
+file, line 1: Parse error before numeric literal
+file, line 2: Parse error before "_"
+file, line 3: Parse error before "a"