From 2cdc43c0282ca3cc5bcee97ff016b046daf42074 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 12 Dec 2017 16:40:10 -0800 Subject: [PATCH 1/5] When a module has errors but the interface hash is the same, keep the cache Closes #4212. However, there are still many scenarios where a trivial error affects the interface hash and hence causes a recheck of everything downstream. Also, this affects regular incremental mode too (making it faster, but if there's a bug in my logic, it will also be hit by that bug.) --- mypy/build.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 21fde0239c16..d88a82f4ac77 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1139,6 +1139,20 @@ def compute_hash(text: str) -> str: return hashlib.md5(text.encode('utf-8')).hexdigest() +def compute_interface_hash(tree: MypyFile, manager: BuildManager) -> str: + """Compute interface hash for a tree. + + Should be called when we want to interface hash but don't want to + write the cache. + """ + data = tree.serialize() + if manager.options.debug_cache: + data_str = json.dumps(data, indent=2, sort_keys=True) + else: + data_str = json.dumps(data, sort_keys=True) + return compute_hash(data_str) + + def write_cache(id: str, path: str, tree: MypyFile, dependencies: List[str], suppressed: List[str], child_modules: List[str], dep_prios: List[int], @@ -1952,9 +1966,12 @@ def write_cache(self) -> None: else: is_errors = self.transitive_error if is_errors: - delete_cache(self.id, self.path, self.manager) - self.meta = None - self.mark_interface_stale(on_errors=True) + new_interface_hash = compute_interface_hash(self.tree, self.manager) + if new_interface_hash != self.interface_hash: + delete_cache(self.id, self.path, self.manager) + self.meta = None + self.mark_interface_stale(on_errors=True) + self.interface_hash = new_interface_hash return dep_prios = self.dependency_priorities() new_interface_hash, self.meta = write_cache( From 438acd94f093cf2a044f9cc8ebacc0fa4b2cf9a3 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 12 Dec 2017 17:20:22 -0800 Subject: [PATCH 2/5] Fix testIncrementalInternalFunctionDefinitionChange --- test-data/unit/check-incremental.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 6df0d81a686b..b71a8be9d78e 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -201,7 +201,7 @@ def foo() -> int: return "foo" return inner2() -[rechecked mod1, mod2] +[rechecked mod2] [stale] [out2] tmp/mod2.py:4: error: Incompatible return value type (got "str", expected "int") From 2d684d216e46746449934252f7eeb2333642835c Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 12 Dec 2017 17:28:15 -0800 Subject: [PATCH 3/5] Fix testCacheDeletedAfterErrorsFound3 --- mypy/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index d88a82f4ac77..868190b83d0c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1967,8 +1967,8 @@ def write_cache(self) -> None: is_errors = self.transitive_error if is_errors: new_interface_hash = compute_interface_hash(self.tree, self.manager) + delete_cache(self.id, self.path, self.manager) if new_interface_hash != self.interface_hash: - delete_cache(self.id, self.path, self.manager) self.meta = None self.mark_interface_stale(on_errors=True) self.interface_hash = new_interface_hash From b04bbc548acfdf19be4341a9f2df487c094f0bc0 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 12 Dec 2017 20:58:20 -0800 Subject: [PATCH 4/5] Different idea for fixing testCacheDeletedAfterErrorsFound3 --- mypy/build.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 868190b83d0c..b6a64c36ec3c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1272,7 +1272,8 @@ def write_cache(id: str, path: str, tree: MypyFile, return interface_hash, cache_meta_from_dict(meta, data_json) -def delete_cache(id: str, path: str, manager: BuildManager) -> None: +def delete_cache(id: str, path: str, manager: BuildManager, + keep_in_memory: bool = False) -> None: """Delete cache files for a module. The cache files for a module are deleted when mypy finds errors there. @@ -1282,7 +1283,7 @@ def delete_cache(id: str, path: str, manager: BuildManager) -> None: path = os.path.abspath(path) meta_json, data_json = get_cache_names(id, path, manager) manager.log('Deleting {} {} {} {}'.format(id, path, meta_json, data_json)) - if id in manager.saved_cache: + if id in manager.saved_cache and not keep_in_memory: del manager.saved_cache[id] for filename in [data_json, meta_json]: @@ -1967,11 +1968,13 @@ def write_cache(self) -> None: is_errors = self.transitive_error if is_errors: new_interface_hash = compute_interface_hash(self.tree, self.manager) - delete_cache(self.id, self.path, self.manager) if new_interface_hash != self.interface_hash: + delete_cache(self.id, self.path, self.manager) self.meta = None self.mark_interface_stale(on_errors=True) self.interface_hash = new_interface_hash + else: + delete_cache(self.id, self.path, self.manager, keep_in_memory=True) return dep_prios = self.dependency_priorities() new_interface_hash, self.meta = write_cache( From 6ff3293a64ce48095c1808861e846b1dbaa703ee Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 13 Dec 2017 16:43:24 -0800 Subject: [PATCH 5/5] Another attempt -- but I already know it's broken. --- mypy/build.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index b6a64c36ec3c..32a8ac302255 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1272,8 +1272,7 @@ def write_cache(id: str, path: str, tree: MypyFile, return interface_hash, cache_meta_from_dict(meta, data_json) -def delete_cache(id: str, path: str, manager: BuildManager, - keep_in_memory: bool = False) -> None: +def delete_cache(id: str, path: str, manager: BuildManager) -> None: """Delete cache files for a module. The cache files for a module are deleted when mypy finds errors there. @@ -1283,7 +1282,7 @@ def delete_cache(id: str, path: str, manager: BuildManager, path = os.path.abspath(path) meta_json, data_json = get_cache_names(id, path, manager) manager.log('Deleting {} {} {} {}'.format(id, path, meta_json, data_json)) - if id in manager.saved_cache and not keep_in_memory: + if id in manager.saved_cache: del manager.saved_cache[id] for filename in [data_json, meta_json]: @@ -1962,8 +1961,9 @@ def write_cache(self) -> None: assert self.tree is not None, "Internal error: method must be called on parsed file only" if not self.path or self.options.cache_dir == os.devnull: return + is_errors_for_file = self.manager.errors.is_errors_for_file(self.path) if self.manager.options.quick_and_dirty: - is_errors = self.manager.errors.is_errors_for_file(self.path) + is_errors = is_errors_for_file else: is_errors = self.transitive_error if is_errors: @@ -1973,9 +1973,11 @@ def write_cache(self) -> None: self.meta = None self.mark_interface_stale(on_errors=True) self.interface_hash = new_interface_hash - else: - delete_cache(self.id, self.path, self.manager, keep_in_memory=True) - return + return + if is_errors_for_file: + delete_cache(self.id, self.path, self.manager) + self.meta = None + return dep_prios = self.dependency_priorities() new_interface_hash, self.meta = write_cache( self.id, self.path, self.tree,