Extend tests to check if mypy re-checks specific files during incremental mode (#1811)

* Allow tests to check for specific file changes

This commit adds an [stale ...] annotation which, when added to
incremental tests, will make sure that only the files listed in the
annotation are rechecked by mypy during the second run in incremental
mode.

Author: Michael0x2a

mypy/build.py

@@ -317,6 +317,7 @@ class BuildManager:
       errors:          Used for reporting all errors
       options:         Build options
       missing_modules: Set of modules that could not be imported encountered so far
+      stale_modules:   Set of modules that needed to be rechecked
     """
 
     def __init__(self, data_dir: str,
@@ -338,6 +339,7 @@ def __init__(self, data_dir: str,
         self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors)
         self.type_checker = TypeChecker(self.errors, self.modules, options=options)
         self.missing_modules = set()  # type: Set[str]
+        self.stale_modules = set()  # type: Set[str]
 
     def all_imported_modules_in_file(self,
                                      file: MypyFile) -> List[Tuple[int, str, int]]:
@@ -1145,6 +1147,7 @@ def is_fresh(self) -> bool:
     def mark_stale(self) -> None:
         """Throw away the cache data for this file, marking it as stale."""
         self.meta = None
+        self.manager.stale_modules.add(self.id)
 
     def check_blockers(self) -> None:
         """Raise CompileError if a blocking error is detected."""

mypy/test/data.py

@@ -6,7 +6,7 @@
 from os import remove, rmdir
 import shutil
 
-from typing import Callable, List, Tuple
+from typing import Callable, List, Tuple, Set, Optional
 
 from mypy.myunit import TestCase, SkipTestCaseException
 
@@ -41,6 +41,7 @@ def parse_test_cases(
             i += 1
 
             files = []  # type: List[Tuple[str, str]] # path and contents
+            stale_modules = None  # type: Optional[Set[str]]  # module names
             while i < len(p) and p[i].id not in ['out', 'case']:
                 if p[i].id == 'file':
                     # Record an extra file needed for the test case.
@@ -57,6 +58,11 @@ def parse_test_cases(
                         fnam = '__builtin__.py'
                     files.append((os.path.join(base_path, fnam), f.read()))
                     f.close()
+                elif p[i].id == 'stale':
+                    if p[i].arg is None:
+                        stale_modules = set()
+                    else:
+                        stale_modules = {item.strip() for item in p[i].arg.split(',')}
                 else:
                     raise ValueError(
                         'Invalid section header {} in {} at line {}'.format(
@@ -78,7 +84,8 @@ def parse_test_cases(
                 expand_errors(input, tcout, 'main')
                 lastline = p[i].line if i < len(p) else p[i - 1].line + 9999
                 tc = DataDrivenTestCase(p[i0].arg, input, tcout, path,
-                                        p[i0].line, lastline, perform, files)
+                                        p[i0].line, lastline, perform,
+                                        files, stale_modules)
                 out.append(tc)
         if not ok:
             raise ValueError(
@@ -99,11 +106,12 @@ class DataDrivenTestCase(TestCase):
 
     # (file path, file content) tuples
     files = None  # type: List[Tuple[str, str]]
+    expected_stale_modules = None  # type: Optional[Set[str]]
 
     clean_up = None  # type: List[Tuple[bool, str]]
 
     def __init__(self, name, input, output, file, line, lastline,
-                 perform, files):
+                 perform, files, expected_stale_modules):
         super().__init__(name)
         self.input = input
         self.output = output
@@ -112,6 +120,7 @@ def __init__(self, name, input, output, file, line, lastline,
         self.line = line
         self.perform = perform
         self.files = files
+        self.expected_stale_modules = expected_stale_modules
 
     def set_up(self) -> None:
         super().set_up()

mypy/test/testcheck.py

@@ -4,6 +4,7 @@
 import re
 import shutil
 import sys
+import time
 
 from typing import Tuple, List, Dict, Set
 
@@ -80,8 +81,10 @@ def run_test(self, testcase: DataDrivenTestCase) -> None:
         if incremental:
             # Incremental tests are run once with a cold cache, once with a warm cache.
             # Expect success on first run, errors from testcase.output (if any) on second run.
+            # We briefly sleep to make sure file timestamps are distinct.
             self.clear_cache()
             self.run_test_once(testcase, 1)
+            time.sleep(0.1)
             self.run_test_once(testcase, 2)
         elif optional:
             try:
@@ -145,6 +148,11 @@ def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
 
         if incremental and res:
             self.verify_cache(module_name, program_name, a, res.manager)
+            if testcase.expected_stale_modules is not None and incremental == 2:
+                assert_string_arrays_equal(
+                    list(sorted(testcase.expected_stale_modules)),
+                    list(sorted(res.manager.stale_modules.difference({"__main__"}))),
+                    'Set of stale modules does not match expected set')
 
     def verify_cache(self, module_name: str, program_name: str, a: List[str],
                      manager: build.BuildManager) -> None:

test-data/unit/check-incremental.test

@@ -3,6 +3,12 @@
 -- The first time it must pass.
 -- Before it is run the second time, any *.py.next files are copied to *.py.
 -- The second time it must produce the errors given in the [out] section, if any.
+--
+-- Any files that we expect to be stale and rechecked should be annotated in
+-- the [stale] annotation. The test suite will automatically assume that
+-- __main__ is stale so we can avoid constantly having to annotate it.
+-- The list of stale files can be in any arbitrary order, or can be left empty
+-- if no files should be stale.
 
 [case testIncrementalEmpty]
 [out]
@@ -15,6 +21,7 @@ def foo():
 [file m.py.next]
 def foo() -> None:
     pass
+[stale m]
 [out]
 
 [case testIncrementalError]
@@ -25,7 +32,66 @@ def foo() -> None:
 [file m.py.next]
 def foo() -> None:
     bar()
+[stale m]
 [out]
 main:1: note: In module imported here:
 tmp/m.py: note: In function "foo":
 tmp/m.py:2: error: Name 'bar' is not defined
+
+[case testIncrementalSimpleImportSequence]
+import mod1
+mod1.func1()
+
+[file mod1.py]
+import mod2
+def func1() -> None: mod2.func2()
+
+[file mod2.py]
+import mod3
+def func2() -> None: mod3.func3()
+
+[file mod3.py]
+def func3() -> None: pass
+
+[stale]
+[out]
+
+
+[case testIncrementalSimpleImportCascade]
+import mod1
+mod1.func1()
+
+[file mod1.py]
+import mod2
+def func1() -> None: mod2.func2()
+
+[file mod2.py]
+import mod3
+def func2() -> None: mod3.func3()
+
+[file mod3.py]
+def func3() -> None: pass
+
+[file mod3.py.next]
+def func3() -> int: return 2
+
+[stale mod1, mod2, mod3]
+[out]
+
+[case testIncrementalSimpleBranchingModules]
+import mod1
+import mod2
+
+[file mod1.py]
+def func() -> None: pass
+
+[file mod2.py]
+def func() -> None: pass
+
+[file mod1.py.next]
+def func() -> int: return 1
+
+[stale mod1]
+[out]
+
+