From 2953917d2cf513e22d2b10b6da7c7d55f79c7801 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 6 Jul 2016 13:14:34 -0700 Subject: [PATCH 1/4] Allow tests to check for specific file changes This commit adds an [expectStale ...] annotation which, when added to incremental tests, will make sure that only the files listed in the annotation are rechecked by mypy during the second run in incremental mode. --- mypy/build.py | 5 ++ mypy/test/data.py | 12 +++- mypy/test/testcheck.py | 5 ++ test-data/unit/check-incremental.test | 86 +++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 0bbded90adde..5fb68538d6bb 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -55,6 +55,7 @@ class BuildResult: files: Dictionary from module name to related AST node. types: Dictionary from parse tree node to its inferred type. errors: List of error messages. + stale: Set of stale modules that were rechecked """ def __init__(self, manager: 'BuildManager') -> None: @@ -62,6 +63,7 @@ def __init__(self, manager: 'BuildManager') -> None: self.files = manager.modules self.types = manager.type_checker.type_map self.errors = manager.errors.messages() + self.stale = manager.stale_modules class BuildSource: @@ -317,6 +319,7 @@ class BuildManager: errors: Used for reporting all errors options: Build options missing_modules: Set of modules that could not be imported encountered so far + stale_modules: Set of modules that needed to be rechecked """ def __init__(self, data_dir: str, @@ -338,6 +341,7 @@ def __init__(self, data_dir: str, self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors) self.type_checker = TypeChecker(self.errors, self.modules, options=options) self.missing_modules = set() # type: Set[str] + self.stale_modules = set() # type: Set[str] def all_imported_modules_in_file(self, file: MypyFile) -> List[Tuple[int, str, int]]: @@ -1145,6 +1149,7 @@ def is_fresh(self) -> bool: def mark_stale(self) -> None: """Throw away the cache data for this file, marking it as stale.""" self.meta = None + self.manager.stale_modules.add(self.id) def check_blockers(self) -> None: """Raise CompileError if a blocking error is detected.""" diff --git a/mypy/test/data.py b/mypy/test/data.py index bc5b7c40fc0d..93aea4d70c0f 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -6,7 +6,7 @@ from os import remove, rmdir import shutil -from typing import Callable, List, Tuple +from typing import Callable, List, Tuple, Set from mypy.myunit import TestCase, SkipTestCaseException @@ -41,6 +41,7 @@ def parse_test_cases( i += 1 files = [] # type: List[Tuple[str, str]] # path and contents + stale_modules = set(["__main__"]) # type: Set[str] # module names while i < len(p) and p[i].id not in ['out', 'case']: if p[i].id == 'file': # Record an extra file needed for the test case. @@ -57,6 +58,8 @@ def parse_test_cases( fnam = '__builtin__.py' files.append((os.path.join(base_path, fnam), f.read())) f.close() + elif p[i].id in ('expectStale', 'expectstale'): + stale_modules.update({item.strip() for item in p[i].arg.split(',')}) else: raise ValueError( 'Invalid section header {} in {} at line {}'.format( @@ -78,7 +81,8 @@ def parse_test_cases( expand_errors(input, tcout, 'main') lastline = p[i].line if i < len(p) else p[i - 1].line + 9999 tc = DataDrivenTestCase(p[i0].arg, input, tcout, path, - p[i0].line, lastline, perform, files) + p[i0].line, lastline, perform, + files, stale_modules) out.append(tc) if not ok: raise ValueError( @@ -99,11 +103,12 @@ class DataDrivenTestCase(TestCase): # (file path, file content) tuples files = None # type: List[Tuple[str, str]] + expected_stale_modules = None # type: Set[str] clean_up = None # type: List[Tuple[bool, str]] def __init__(self, name, input, output, file, line, lastline, - perform, files): + perform, files, expected_stale_modules): super().__init__(name) self.input = input self.output = output @@ -112,6 +117,7 @@ def __init__(self, name, input, output, file, line, lastline, self.line = line self.perform = perform self.files = files + self.expected_stale_modules = expected_stale_modules def set_up(self) -> None: super().set_up() diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 0c9ed87cdc18..035053d4b546 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -145,6 +145,11 @@ def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None: if incremental and res: self.verify_cache(module_name, program_name, a, res.manager) + if incremental == 2: + assert_string_arrays_equal( + list(sorted(testcase.expected_stale_modules)), + list(sorted(res.stale)), + 'Set of stale modules does not match expected set') def verify_cache(self, module_name: str, program_name: str, a: List[str], manager: build.BuildManager) -> None: diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index c27bcd71e4a8..0835260d0ef6 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -3,6 +3,12 @@ -- The first time it must pass. -- Before it is run the second time, any *.py.next files are copied to *.py. -- The second time it must produce the errors given in the [out] section, if any. +-- +-- Any files that we expect to be stale and rechecked should be annotated in +-- the [expectStale] section. The test suite will automatically assume that +-- __main__ is stale so we can avoid constantly having to annotate it. +-- The list of stale files can be in any arbitrary order. You can have multiple +-- [expectStale] annotations per case: the test suite will union them together. [case testIncrementalEmpty] [out] @@ -15,6 +21,7 @@ def foo(): [file m.py.next] def foo() -> None: pass +[expectStale m] [out] [case testIncrementalError] @@ -25,7 +32,86 @@ def foo() -> None: [file m.py.next] def foo() -> None: bar() +[expectStale m] [out] main:1: note: In module imported here: tmp/m.py: note: In function "foo": tmp/m.py:2: error: Name 'bar' is not defined + +[case testIncrementalSimpleImportSequence] +import mod1 +mod1.func1() + +[file mod1.py] +import mod2 +def func1() -> None: mod2.func2() + +[file mod2.py] +import mod3 +def func2() -> None: mod3.func3() + +[file mod3.py] +def func3() -> None: pass + +[file mod1.py.next] +import mod2 +def func1() -> None: mod2.func2() + +[file mod2.py.next] +import mod3 +def func2() -> None: mod3.func3() + +[file mod3.py.next] +def func3() -> None: pass + +[out] + + +[case testIncrementalSimpleImportCascade] +import mod1 +mod1.func1() + +[file mod1.py] +import mod2 +def func1() -> None: mod2.func2() + +[file mod2.py] +import mod3 +def func2() -> None: mod3.func3() + +[file mod3.py] +def func3() -> None: pass + +[file mod1.py.next] +import mod2 +def func1() -> None: mod2.func2() + +[file mod2.py.next] +import mod3 +def func2() -> None: mod3.func3() + +[file mod3.py.next] +def func3() -> int: return 2 + +[expectStale mod1, mod2, mod3] +[out] + +[case testIncrementalSimpleBranchingModules] +import mod1 +import mod2 + +[file mod1.py] +def func() -> None: pass + +[file mod2.py] +def func() -> None: pass + +[file mod1.py.next] +def func() -> int: return 1 + +[file mod2.py.next] +def func() -> None: pass + +[expectStale mod1] +[out] + From 5124d8e25f4e5c0e9cf4040c6b6558cf901f138b Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 6 Jul 2016 15:48:22 -0700 Subject: [PATCH 2/4] Remove unnecessary file changes --- test-data/unit/check-incremental.test | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 0835260d0ef6..868cbd2830cb 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -53,17 +53,6 @@ def func2() -> None: mod3.func3() [file mod3.py] def func3() -> None: pass -[file mod1.py.next] -import mod2 -def func1() -> None: mod2.func2() - -[file mod2.py.next] -import mod3 -def func2() -> None: mod3.func3() - -[file mod3.py.next] -def func3() -> None: pass - [out] @@ -82,14 +71,6 @@ def func2() -> None: mod3.func3() [file mod3.py] def func3() -> None: pass -[file mod1.py.next] -import mod2 -def func1() -> None: mod2.func2() - -[file mod2.py.next] -import mod3 -def func2() -> None: mod3.func3() - [file mod3.py.next] def func3() -> int: return 2 From b255d6e0e98d928c85197ef37aec388073412356 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 6 Jul 2016 16:32:43 -0700 Subject: [PATCH 3/4] Respond to comments, try adding short sleep --- mypy/build.py | 2 -- mypy/test/data.py | 11 +++++++---- mypy/test/testcheck.py | 7 +++++-- test-data/unit/check-incremental.test | 19 +++++++++---------- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 5fb68538d6bb..1bd396fb89d8 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -55,7 +55,6 @@ class BuildResult: files: Dictionary from module name to related AST node. types: Dictionary from parse tree node to its inferred type. errors: List of error messages. - stale: Set of stale modules that were rechecked """ def __init__(self, manager: 'BuildManager') -> None: @@ -63,7 +62,6 @@ def __init__(self, manager: 'BuildManager') -> None: self.files = manager.modules self.types = manager.type_checker.type_map self.errors = manager.errors.messages() - self.stale = manager.stale_modules class BuildSource: diff --git a/mypy/test/data.py b/mypy/test/data.py index 93aea4d70c0f..ea634ea004a5 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -41,7 +41,7 @@ def parse_test_cases( i += 1 files = [] # type: List[Tuple[str, str]] # path and contents - stale_modules = set(["__main__"]) # type: Set[str] # module names + stale_modules = None # type: Optional[Set[str]] # module names while i < len(p) and p[i].id not in ['out', 'case']: if p[i].id == 'file': # Record an extra file needed for the test case. @@ -58,8 +58,11 @@ def parse_test_cases( fnam = '__builtin__.py' files.append((os.path.join(base_path, fnam), f.read())) f.close() - elif p[i].id in ('expectStale', 'expectstale'): - stale_modules.update({item.strip() for item in p[i].arg.split(',')}) + elif p[i].id == 'stale': + if p[i].arg is None: + stale_modules = set() + else: + stale_modules = {item.strip() for item in p[i].arg.split(',')} else: raise ValueError( 'Invalid section header {} in {} at line {}'.format( @@ -103,7 +106,7 @@ class DataDrivenTestCase(TestCase): # (file path, file content) tuples files = None # type: List[Tuple[str, str]] - expected_stale_modules = None # type: Set[str] + expected_stale_modules = None # type: Optional[Set[str]] clean_up = None # type: List[Tuple[bool, str]] diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 035053d4b546..3d8457095ab2 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -4,6 +4,7 @@ import re import shutil import sys +import time from typing import Tuple, List, Dict, Set @@ -80,8 +81,10 @@ def run_test(self, testcase: DataDrivenTestCase) -> None: if incremental: # Incremental tests are run once with a cold cache, once with a warm cache. # Expect success on first run, errors from testcase.output (if any) on second run. + # We briefly sleep to make sure file timestamps are distinct. self.clear_cache() self.run_test_once(testcase, 1) + time.sleep(0.1) self.run_test_once(testcase, 2) elif optional: try: @@ -145,10 +148,10 @@ def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None: if incremental and res: self.verify_cache(module_name, program_name, a, res.manager) - if incremental == 2: + if testcase.expected_stale_modules is not None and incremental == 2: assert_string_arrays_equal( list(sorted(testcase.expected_stale_modules)), - list(sorted(res.stale)), + list(sorted(res.manager.stale_modules.difference({"__main__"}))), 'Set of stale modules does not match expected set') def verify_cache(self, module_name: str, program_name: str, a: List[str], diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 868cbd2830cb..86c1167b05f6 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -5,10 +5,10 @@ -- The second time it must produce the errors given in the [out] section, if any. -- -- Any files that we expect to be stale and rechecked should be annotated in --- the [expectStale] section. The test suite will automatically assume that +-- the [stale] annotation. The test suite will automatically assume that -- __main__ is stale so we can avoid constantly having to annotate it. --- The list of stale files can be in any arbitrary order. You can have multiple --- [expectStale] annotations per case: the test suite will union them together. +-- The list of stale files can be in any arbitrary order, or can be left empty +-- if no files should be stale. [case testIncrementalEmpty] [out] @@ -21,7 +21,7 @@ def foo(): [file m.py.next] def foo() -> None: pass -[expectStale m] +[stale m] [out] [case testIncrementalError] @@ -32,7 +32,7 @@ def foo() -> None: [file m.py.next] def foo() -> None: bar() -[expectStale m] +[stale m] [out] main:1: note: In module imported here: tmp/m.py: note: In function "foo": @@ -53,6 +53,7 @@ def func2() -> None: mod3.func3() [file mod3.py] def func3() -> None: pass +[stale] [out] @@ -74,7 +75,7 @@ def func3() -> None: pass [file mod3.py.next] def func3() -> int: return 2 -[expectStale mod1, mod2, mod3] +[stale mod1, mod2, mod3] [out] [case testIncrementalSimpleBranchingModules] @@ -90,9 +91,7 @@ def func() -> None: pass [file mod1.py.next] def func() -> int: return 1 -[file mod2.py.next] -def func() -> None: pass - -[expectStale mod1] +[stale mod1] [out] + From 89763f56cf4695ba36643f5c3a2d302f2de8e9f5 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 6 Jul 2016 16:51:27 -0700 Subject: [PATCH 4/4] Add missing import --- mypy/test/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/data.py b/mypy/test/data.py index ea634ea004a5..9881ed27f40a 100644 --- a/mypy/test/data.py +++ b/mypy/test/data.py @@ -6,7 +6,7 @@ from os import remove, rmdir import shutil -from typing import Callable, List, Tuple, Set +from typing import Callable, List, Tuple, Set, Optional from mypy.myunit import TestCase, SkipTestCaseException