Recompute lib_path on every fine-grained incremental run (#4981)

This fixes some crashes and false positives when creating or removing
__init__.py files that do not appear in the build.

This just factors out the existing implementation of lib_path
computation and reuses it. It is somewhat inefficient and should be
improved on if we are going to call it on every fine-grained update.

Author: msullivan

mypy/build.py

@@ -195,19 +195,11 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None:
         raise
 
 
-def _build(sources: List[BuildSource],
-           options: Options,
-           alt_lib_path: Optional[str],
-           bin_dir: Optional[str],
-           flush_errors: Callable[[List[str], bool], None],
-           fscache: Optional[FileSystemCache],
-           ) -> BuildResult:
-    # This seems the most reasonable place to tune garbage collection.
-    gc.set_threshold(50000)
-
-    data_dir = default_data_dir(bin_dir)
-    fscache = fscache or FileSystemCache()
-
+def compute_lib_path(sources: List[BuildSource],
+                     options: Options,
+                     alt_lib_path: Optional[str],
+                     data_dir: str,
+                     fscache: FileSystemCache) -> List[str]:
     # Determine the default module search path.
     lib_path = default_lib_path(data_dir,
                                 options.python_version,
@@ -219,7 +211,9 @@ def _build(sources: List[BuildSource],
         # as in the source tree.
         root_dir = dirname(dirname(__file__))
         lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub'))
-    else:
+    # alt_lib_path is used by some tests to bypass the normal lib_path mechanics.
+    # If we don't have one, grab directories of source files.
+    if not alt_lib_path:
         for source in sources:
             if source.path:
                 # Include directory of the program file in the module search path.
@@ -246,6 +240,24 @@ def _build(sources: List[BuildSource],
     if alt_lib_path:
         lib_path.insert(0, alt_lib_path)
 
+    return lib_path
+
+
+def _build(sources: List[BuildSource],
+           options: Options,
+           alt_lib_path: Optional[str],
+           bin_dir: Optional[str],
+           flush_errors: Callable[[List[str], bool], None],
+           fscache: Optional[FileSystemCache],
+           ) -> BuildResult:
+    # This seems the most reasonable place to tune garbage collection.
+    gc.set_threshold(50000)
+
+    data_dir = default_data_dir(bin_dir)
+    fscache = fscache or FileSystemCache()
+
+    lib_path = compute_lib_path(sources, options, alt_lib_path, data_dir, fscache)
+
     reports = Reports(data_dir, options.report_dirs)
     source_set = BuildSourceSet(sources)
     errors = Errors(options.show_error_context, options.show_column_numbers)

mypy/dmypy_server.py

@@ -309,14 +309,19 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
 
     def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
         assert self.fine_grained_manager is not None
+        manager = self.fine_grained_manager.manager
 
         t0 = time.time()
         self.update_sources(sources)
         changed, removed = self.find_changed(sources)
+        # Update the lib_path, which can change when sources do.
+        # TODO: This is slow.
+        manager.lib_path = tuple(mypy.build.compute_lib_path(
+            sources, manager.options, self.alt_lib_path, manager.data_dir, self.fscache))
         t1 = time.time()
         messages = self.fine_grained_manager.update(changed, removed)
         t2 = time.time()
-        self.fine_grained_manager.manager.log(
+        manager.log(
             "fine-grained increment: find_changed: {:.3f}s, update: {:.3f}s".format(
                 t1 - t0, t2 - t1))
         status = 1 if messages else 0

mypy/test/testerrorstream.py

@@ -40,7 +40,6 @@ def flush_errors(msgs: List[str], serious: bool) -> None:
     try:
         build.build(sources=sources,
                     options=options,
-                    alt_lib_path=test_temp_dir,
                     flush_errors=flush_errors)
     except CompileError as e:
         assert e.messages == []

mypy/test/testfinegrained.py

@@ -5,6 +5,11 @@
 
 See the comment at the top of test-data/unit/fine-grained.test for more
 information.
+
+N.B.: Unlike most of the other test suites, testfinegrained does not
+rely on an alt_lib_path for finding source files. This means that they
+can test interactions with the lib_path that is built implicitly based
+on specified sources.
 """
 
 import os
@@ -84,7 +89,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             config = None
         if config:
             parse_config_file(options, config)
-        server = Server(options, alt_lib_path=test_temp_dir)
+        server = Server(options)
 
         step = 1
         sources = self.parse_sources(main_src, step, options)
@@ -186,8 +191,7 @@ def build(self,
               sources: List[BuildSource]) -> List[str]:
         try:
             result = build.build(sources=sources,
-                                 options=options,
-                                 alt_lib_path=test_temp_dir)
+                                 options=options)
         except CompileError as e:
             return e.messages
         return result.errors

test-data/unit/fine-grained-modules.test

@@ -484,6 +484,28 @@ def f(x: int) -> None: pass
 [out]
 ==
 
+[case testDeleteSubpackageInit1]
+# cmd: mypy q/r/s.py
+# flags: --follow-imports=skip --ignore-missing-imports
+[file q/__init__.py]
+[file q/r/__init__.py]
+[file q/r/s.py]
+[delete q/__init__.py.2]
+[out]
+==
+
+[case testAddSubpackageInit2]
+# cmd: mypy q/r/s.py
+# flags: --follow-imports=skip --ignore-missing-imports
+[file q/r/__init__.py]
+[file q/r/s.py]
+1
+[file q/r/s.py.2]
+2
+[file q/__init__.py.2]
+[out]
+==
+
 [case testModifyTwoFilesNoError2]
 import a
 [file a.py]