Merge branch 'ignore-multiline'

Fixes #627.

Author: JukkaL

mypy/lex.py

@@ -157,14 +157,16 @@ def __str__(self):
 INVALID_DEDENT = 5
 
 
-def lex(string: Union[str, bytes], first_line: int = 1, pyversion: int = 3) -> List[Token]:
-    """Analyze string and return an array of token objects.
+def lex(string: Union[str, bytes], first_line: int = 1,
+        pyversion: int = 3) -> Tuple[List[Token], Set[int]]:
+    """Analyze string, and return an array of token objects and the lines to ignore.
 
-    The last token is always Eof.
+    The last token is always Eof. The intention is to ignore any
+    semantic and type check errors on the ignored lines.
     """
     l = Lexer(pyversion)
     l.lex(string, first_line)
-    return l.tok
+    return l.tok, l.ignored_lines
 
 
 # Reserved words (not including operators)
@@ -291,12 +293,16 @@ class Lexer:
 
     pyversion = 3
 
+    # Lines to ignore (using # type: ignore).
+    ignored_lines = Undefined(Set[int])
+
     def __init__(self, pyversion: int = 3) -> None:
         self.map = [self.unknown_character] * 256
         self.tok = []
         self.indents = [0]
         self.open_brackets = []
         self.pyversion = pyversion
+        self.ignored_lines = set()
         # Fill in the map from valid character codes to relevant lexer methods.
         for seq, method in [('ABCDEFGHIJKLMNOPQRSTUVWXYZ', self.lex_name),
                             ('abcdefghijklmnopqrstuvwxyz_', self.lex_name),
@@ -811,6 +817,8 @@ def add_pre_whitespace(self, s: str) -> None:
         self.pre_whitespace += s
         self.i += len(s)
 
+    type_ignore_exp = re.compile(r'[ \t]*#[ \t]*type:[ \t]*ignore\b')
+
     def add_token(self, tok: Token) -> None:
         """Store a token.
 
@@ -823,6 +831,11 @@ def add_token(self, tok: Token) -> None:
                 and not isinstance(tok, Dedent)):
             raise ValueError('Empty token')
         tok.pre = self.pre_whitespace
+        if self.type_ignore_exp.match(tok.pre):
+            delta = 0
+            if '\n' in tok.pre or '\r' in tok.pre:
+                delta += 1
+            self.ignored_lines.add(self.line - delta)
         tok.line = self.line
         self.tok.append(tok)
         self.i += len(tok.string)

mypy/parse.py

@@ -112,11 +112,10 @@ def __init__(self, fnam: str, errors: Errors, pyversion: int,
             self.errors.set_file('<input>')
 
     def parse(self, s: Union[str, bytes]) -> MypyFile:
-        self.tok = lex.lex(s, pyversion=self.pyversion)
+        self.tok, self.ignored_lines = lex.lex(s, pyversion=self.pyversion)
         self.ind = 0
         self.imports = []
         self.future_options = []
-        self.ignored_lines = set()
         file = self.parse_file()
         if self.raise_on_error and self.errors.is_errors():
             self.errors.raise_error()
@@ -1564,13 +1563,8 @@ def expect_type(self, typ: type) -> Token:
     def expect_colon_and_break(self) -> Tuple[Token, Token]:
         return self.expect_type(Colon), self.expect_type(Break)
 
-    type_ignore_exp = re.compile(r'[ \t]*#[ \t]*type:[ \t]*ignore\b')
-
     def expect_break(self) -> Token:
-        token = self.expect_type(Break)
-        if self.type_ignore_exp.match(token.pre):
-            self.ignored_lines.add(token.line)
-        return token
+        return self.expect_type(Break)
 
     def current(self) -> Token:
         return self.tok[self.ind]
@@ -1645,7 +1639,7 @@ def parse_type_comment(self, token: Token, signature: bool) -> Type:
             if type_as_str == 'ignore':
                 # Actually a "# type: ignore" annotation -> not a type.
                 return None
-            tokens = lex.lex(type_as_str, token.line)
+            tokens = lex.lex(type_as_str, token.line)[0]
             if len(tokens) < 2:
                 # Empty annotation (only Eof token)
                 self.errors.report(token.line, 'Empty type annotation')

mypy/parsetype.py

@@ -189,7 +189,7 @@ def parse_str_as_type(typestr: str, line: int) -> Type:
     """
 
     typestr = typestr.strip()
-    tokens = lex(typestr, line)
+    tokens = lex(typestr, line)[0]
     result, i = parse_type(tokens, 0)
     if i < len(tokens) - 2:
         raise TypeParseError(tokens[i], i)

mypy/test/data/check-ignore.test

@@ -19,6 +19,14 @@ a.foo
 b()
 1() # E: "int" not callable
 
+[case testIgnoreImportFromErrorMultiline]
+from xyz_m import ( # type: ignore
+    a, b
+)
+a.foo
+b()
+1() # E: "int" not callable
+
 [case testIgnoreImportAllError]
 from xyz_m import * # type: ignore
 x   # E: Name 'x' is not defined

mypy/test/data/parse.test

@@ -3292,3 +3292,33 @@ MypyFile:1(
   ExpressionStmt:2(
     NameExpr(y))
   IgnoredLines(1, 2))
+
+[case testIgnoreAnnotationAndMultilineStatement]
+x = {
+  1: 2  # type: ignore
+}
+y = {   # type: ignore
+  1: 2
+}       # type: ignore
+[out]
+MypyFile:1(
+  AssignmentStmt:1(
+    NameExpr(x)
+    DictExpr:1(
+      IntExpr(1)
+      IntExpr(2)))
+  AssignmentStmt:4(
+    NameExpr(y)
+    DictExpr:4(
+      IntExpr(1)
+      IntExpr(2)))
+  IgnoredLines(2, 4, 6))
+
+[case testIgnoreAnnotationAndMultilineStatement2]
+from m import ( # type: ignore
+  x, y
+)
+[out]
+MypyFile:1(
+  ImportFrom:1(m, [x : x, y : y])
+  IgnoredLines(1))

mypy/test/testlex.py

@@ -425,7 +425,7 @@ def assert_lex(self, src, lexed):
         if lexed.endswith(' ...'):
             lexed = lexed[:-3] + 'Break() Eof()'
 
-        l = lex(src)
+        l = lex(src)[0]
         r = []
         for t in l:
             r.append(str(t))
@@ -439,7 +439,7 @@ def assert_line(self, s, a):
         s = s.replace('\\n', '\n')
         s = s.replace('\\r', '\r')
 
-        tt = lex(s)
+        tt = lex(s)[0]
         r = []
         for t in tt:
             r.append(t.line)