Improvements to blocker handling in fine-grained incremental mode, etc. (#4292)

Handle various additional cases of blocking errors. This also contains a fair
amount of refactoring and minor additional fixes.

Author: JukkaL

mypy/build.py

@@ -90,6 +90,11 @@ def __init__(self, path: Optional[str], module: Optional[str],
         self.module = module or '__main__'
         self.text = text
 
+    def __repr__(self) -> str:
+        return '<BuildSource path=%r module=%r has_text=%s>' % (self.path,
+                                                                self.module,
+                                                                self.text is not None)
+
 
 class BuildSourceSet:
     """Efficiently test a file's membership in the set of build sources."""
@@ -633,7 +638,7 @@ def parse_file(self, id: str, path: str, source: str, ignore_errors: bool) -> My
         Raise CompileError if there is a parse error.
         """
         num_errs = self.errors.num_messages()
-        tree = parse(source, path, self.errors, options=self.options)
+        tree = parse(source, path, id, self.errors, options=self.options)
         tree._fullname = id
         self.add_stats(files_parsed=1,
                        modules_parsed=int(not tree.is_stub),
@@ -2070,7 +2075,11 @@ def dump_graph(graph: Graph) -> None:
 
 
 def load_graph(sources: List[BuildSource], manager: BuildManager) -> Graph:
-    """Given some source files, load the full dependency graph."""
+    """Given some source files, load the full dependency graph.
+
+    As this may need to parse files, this can raise CompileError in case
+    there are syntax errors.
+    """
     graph = {}  # type: Graph
     # The deque is used to implement breadth-first traversal.
     # TODO: Consider whether to go depth-first instead.  This may

mypy/errors.py

@@ -331,6 +331,13 @@ def is_blockers(self) -> bool:
         """Are the any errors that are blockers?"""
         return any(err for err in self.error_info if err.blocker)
 
+    def blocker_module(self) -> Optional[str]:
+        """Return the module with a blocking error, or None if not possible."""
+        for err in self.error_info:
+            if err.blocker:
+                return err.module
+        return None
+
     def is_errors_for_file(self, file: str) -> bool:
         """Are there any errors for the given file?"""
         return file in self.error_files
@@ -340,7 +347,9 @@ def raise_error(self) -> None:
 
         Render the messages suitable for displaying.
         """
-        raise CompileError(self.messages(), use_stdout=True)
+        raise CompileError(self.messages(),
+                           use_stdout=True,
+                           module_with_blocker=self.blocker_module())
 
     def messages(self) -> List[str]:
         """Return a string list that represents the error messages.
@@ -506,11 +515,17 @@ class CompileError(Exception):
 
     messages = None  # type: List[str]
     use_stdout = False
+    # Can be set in case there was a module with a blocking error
+    module_with_blocker = None  # type: Optional[str]
 
-    def __init__(self, messages: List[str], use_stdout: bool = False) -> None:
+    def __init__(self,
+                 messages: List[str],
+                 use_stdout: bool = False,
+                 module_with_blocker: Optional[str] = None) -> None:
         super().__init__('\n'.join(messages))
         self.messages = messages
         self.use_stdout = use_stdout
+        self.module_with_blocker = module_with_blocker
 
 
 class DecodeError(Exception):

mypy/fastparse.py

@@ -68,6 +68,7 @@
 
 def parse(source: Union[str, bytes],
           fnam: str,
+          module: Optional[str],
           errors: Optional[Errors] = None,
           options: Options = Options()) -> MypyFile:
 
@@ -80,7 +81,7 @@ def parse(source: Union[str, bytes],
     if errors is None:
         errors = Errors()
         raise_on_error = True
-    errors.set_file(fnam, None)
+    errors.set_file(fnam, module)
     is_stub_file = fnam.endswith('.pyi')
     try:
         if is_stub_file:
@@ -97,7 +98,7 @@ def parse(source: Union[str, bytes],
         tree.path = fnam
         tree.is_stub = is_stub_file
     except SyntaxError as e:
-        errors.report(e.lineno, e.offset, e.msg)
+        errors.report(e.lineno, e.offset, e.msg, blocker=True)
         tree = MypyFile([], [], False, set())
 
     if raise_on_error and errors.is_errors():
@@ -111,7 +112,7 @@ def parse_type_comment(type_comment: str, line: int, errors: Optional[Errors]) -
         typ = ast3.parse(type_comment, '<type_comment>', 'eval')
     except SyntaxError as e:
         if errors is not None:
-            errors.report(line, e.offset, TYPE_COMMENT_SYNTAX_ERROR)
+            errors.report(line, e.offset, TYPE_COMMENT_SYNTAX_ERROR, blocker=True)
             return None
         else:
             raise
@@ -158,7 +159,7 @@ def __init__(self,
         self.errors = errors
 
     def fail(self, msg: str, line: int, column: int) -> None:
-        self.errors.report(line, column, msg)
+        self.errors.report(line, column, msg, blocker=True)
 
     def generic_visit(self, node: ast3.AST) -> None:
         raise RuntimeError('AST node not implemented: ' + str(type(node)))
@@ -1013,7 +1014,7 @@ def parent(self) -> Optional[ast3.AST]:
 
     def fail(self, msg: str, line: int, column: int) -> None:
         if self.errors:
-            self.errors.report(line, column, msg)
+            self.errors.report(line, column, msg, blocker=True)
 
     def visit_raw_str(self, s: str) -> Type:
         # An escape hatch that allows the AST walker in fastparse2 to

mypy/fastparse2.py

@@ -79,6 +79,7 @@
 
 def parse(source: Union[str, bytes],
           fnam: str,
+          module: Optional[str],
           errors: Optional[Errors] = None,
           options: Options = Options()) -> MypyFile:
     """Parse a source file, without doing any semantic analysis.
@@ -90,7 +91,7 @@ def parse(source: Union[str, bytes],
     if errors is None:
         errors = Errors()
         raise_on_error = True
-    errors.set_file(fnam, None)
+    errors.set_file(fnam, module)
     is_stub_file = fnam.endswith('.pyi')
     try:
         assert options.python_version[0] < 3 and not is_stub_file
@@ -103,7 +104,7 @@ def parse(source: Union[str, bytes],
         tree.path = fnam
         tree.is_stub = is_stub_file
     except SyntaxError as e:
-        errors.report(e.lineno, e.offset, e.msg)
+        errors.report(e.lineno, e.offset, e.msg, blocker=True)
         tree = MypyFile([], [], False, set())
 
     if raise_on_error and errors.is_errors():
@@ -150,7 +151,7 @@ def __init__(self,
         self.errors = errors
 
     def fail(self, msg: str, line: int, column: int) -> None:
-        self.errors.report(line, column, msg)
+        self.errors.report(line, column, msg, blocker=True)
 
     def generic_visit(self, node: ast27.AST) -> None:
         raise RuntimeError('AST node not implemented: ' + str(type(node)))

mypy/parse.py

@@ -7,6 +7,7 @@
 
 def parse(source: Union[str, bytes],
           fnam: str,
+          module: Optional[str],
           errors: Optional[Errors],
           options: Options) -> MypyFile:
     """Parse a source file, without doing any semantic analysis.
@@ -21,11 +22,13 @@ def parse(source: Union[str, bytes],
         import mypy.fastparse
         return mypy.fastparse.parse(source,
                                     fnam=fnam,
+                                    module=module,
                                     errors=errors,
                                     options=options)
     else:
         import mypy.fastparse2
         return mypy.fastparse2.parse(source,
                                      fnam=fnam,
+                                     module=module,
                                      errors=errors,
                                      options=options)