Fine-grained: Apply semantic analyzer patch callbacks (#4658)

This fixes crashes and at least one edge case. Not sure if
all the patch callbacks are necessary when propagating
fine-grained dependencies, but it seems safer to invoke
them all.

Fixes #4657.

Author: JukkaL

mypy/build.py

@@ -35,7 +35,7 @@
 
 from mypy.nodes import (MODULE_REF, MypyFile, Node, ImportBase, Import, ImportFrom, ImportAll)
 from mypy.semanal_pass1 import SemanticAnalyzerPass1
-from mypy.semanal import SemanticAnalyzerPass2
+from mypy.semanal import SemanticAnalyzerPass2, apply_semantic_analyzer_patches
 from mypy.semanal_pass3 import SemanticAnalyzerPass3
 from mypy.checker import TypeChecker
 from mypy.indirection import TypeIndirectionVisitor
@@ -1960,9 +1960,7 @@ def semantic_analysis_pass_three(self) -> None:
         self.patches = patches + self.patches
 
     def semantic_analysis_apply_patches(self) -> None:
-        patches_by_priority = sorted(self.patches, key=lambda x: x[0])
-        for priority, patch_func in patches_by_priority:
-            patch_func()
+        apply_semantic_analyzer_patches(self.patches)
 
     def type_check_first_pass(self) -> None:
         if self.options.semantic_analysis_only:

mypy/semanal.py

@@ -309,24 +309,24 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
             del self.cur_mod_node
             del self.globals
 
-    def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef]) -> None:
+    def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef],
+                        patches: List[Tuple[int, Callable[[], None]]]) -> None:
         """Refresh a stale target in fine-grained incremental mode."""
+        self.patches = patches
         self.scope.enter_file(self.cur_mod_id)
         if isinstance(node, MypyFile):
             self.refresh_top_level(node)
         else:
             self.recurse_into_functions = True
             self.accept(node)
         self.scope.leave()
+        del self.patches
 
     def refresh_top_level(self, file_node: MypyFile) -> None:
         """Reanalyze a stale module top-level in fine-grained incremental mode."""
-        # TODO: Invoke patches in fine-grained incremental mode.
-        self.patches = []
         self.recurse_into_functions = False
         for d in file_node.defs:
             self.accept(d)
-        del self.patches
 
     @contextmanager
     def file_context(self, file_node: MypyFile, fnam: str, options: Options,
@@ -4307,3 +4307,13 @@ def visit_any(self, t: AnyType) -> Type:
         if t.type_of_any == TypeOfAny.explicit:
             return t.copy_modified(TypeOfAny.special_form)
         return t
+
+
+def apply_semantic_analyzer_patches(patches: List[Tuple[int, Callable[[], None]]]) -> None:
+    """Call patch callbacks in the right order.
+
+    This should happen after semantic analyzer pass 3.
+    """
+    patches_by_priority = sorted(patches, key=lambda x: x[0])
+    for priority, patch_func in patches_by_priority:
+        patch_func()

mypy/semanal_pass3.py

@@ -71,8 +71,10 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
         del self.cur_mod_node
         self.patches = []
 
-    def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef]) -> None:
+    def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef],
+                        patches: List[Tuple[int, Callable[[], None]]]) -> None:
         """Refresh a stale target in fine-grained incremental mode."""
+        self.patches = patches
         self.scope.enter_file(self.sem.cur_mod_id)
         if isinstance(node, MypyFile):
             self.recurse_into_functions = False
@@ -81,6 +83,7 @@ def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef]) ->
             self.recurse_into_functions = True
             self.accept(node)
         self.scope.leave()
+        self.patches = []
 
     def refresh_top_level(self, file_node: MypyFile) -> None:
         """Reanalyze a stale module top-level in fine-grained incremental mode."""

mypy/server/aststrip.py

@@ -78,20 +78,24 @@ def strip_file_top_level(self, file_node: MypyFile) -> None:
 
     def visit_class_def(self, node: ClassDef) -> None:
         """Strip class body and type info, but don't strip methods."""
-        node.info.type_vars = []
-        node.info.bases = []
-        node.info.abstract_attributes = []
-        node.info.mro = []
-        node.info.add_type_vars()
-        node.info.tuple_type = None
-        node.info.typeddict_type = None
-        node.info._cache = set()
-        node.info._cache_proper = set()
+        self.strip_type_info(node.info)
         node.base_type_exprs.extend(node.removed_base_type_exprs)
         node.removed_base_type_exprs = []
         with self.enter_class(node.info):
             super().visit_class_def(node)
 
+    def strip_type_info(self, info: TypeInfo) -> None:
+        info.type_vars = []
+        info.bases = []
+        info.abstract_attributes = []
+        info.mro = []
+        info.add_type_vars()
+        info.tuple_type = None
+        info.typeddict_type = None
+        info.tuple_type = None
+        info._cache = set()
+        info._cache_proper = set()
+
     def visit_func_def(self, node: FuncDef) -> None:
         if not self.recurse_into_functions:
             return

mypy/server/update.py

@@ -118,7 +118,9 @@
 """
 
 import os.path
-from typing import Dict, List, Set, Tuple, Iterable, Union, Optional, Mapping, NamedTuple
+from typing import (
+    Dict, List, Set, Tuple, Iterable, Union, Optional, Mapping, NamedTuple, Callable
+)
 
 from mypy.build import (
     BuildManager, State, BuildSource, Graph, load_graph, find_module_clear_caches,
@@ -132,6 +134,7 @@
 )
 from mypy.options import Options
 from mypy.types import Type
+from mypy.semanal import apply_semantic_analyzer_patches
 from mypy.server.astdiff import (
     snapshot_symbol_table, compare_symbol_table_snapshots, is_identical_type, SnapshotItem
 )
@@ -743,6 +746,8 @@ def key(node: DeferredNode) -> int:
         strip_target(deferred.node)
     semantic_analyzer = manager.semantic_analyzer
 
+    patches = []  # type: List[Tuple[int, Callable[[], None]]]
+
     # Second pass of semantic analysis. We don't redo the first pass, because it only
     # does local things that won't go stale.
     for deferred in nodes:
@@ -751,7 +756,7 @@ def key(node: DeferredNode) -> int:
                 fnam=file_node.path,
                 options=manager.options,
                 active_type=deferred.active_typeinfo):
-            manager.semantic_analyzer.refresh_partial(deferred.node)
+            manager.semantic_analyzer.refresh_partial(deferred.node, patches)
 
     # Third pass of semantic analysis.
     for deferred in nodes:
@@ -760,7 +765,9 @@ def key(node: DeferredNode) -> int:
                 fnam=file_node.path,
                 options=manager.options,
                 active_type=deferred.active_typeinfo):
-            manager.semantic_analyzer_pass3.refresh_partial(deferred.node)
+            manager.semantic_analyzer_pass3.refresh_partial(deferred.node, patches)
+
+    apply_semantic_analyzer_patches(patches)
 
     # Merge symbol tables to preserve identities of AST nodes. The file node will remain
     # the same, but other nodes may have been recreated with different identities, such as

test-data/unit/fine-grained.test

@@ -2475,3 +2475,37 @@ else:
 [builtins fixtures/ops.pyi]
 [out]
 ==
+
+[case testNamedTupleWithinFunction]
+from typing import NamedTuple
+import b
+def f() -> None:
+    b.x
+    n = NamedTuple('n', [])
+[file b.py]
+x = 0
+[file b.py.2]
+x = ''
+[out]
+==
+
+[case testNamedTupleFallback]
+# This test will fail without semantic analyzer pass 2 patches
+import a
+[file a.py]
+import b
+[file b.py]
+from typing import NamedTuple
+import c
+c.x
+class N(NamedTuple):
+    count: int
+[file c.py]
+x = 0
+[file c.py.2]
+x = ''
+[builtins fixtures/tuple.pyi]
+[out]
+b.py:5: error: Incompatible types in assignment (expression has type "int", base class "tuple" defined the type as "Callable[[Tuple[int, ...], Any], int]")
+==
+b.py:5: error: Incompatible types in assignment (expression has type "int", base class "tuple" defined the type as "Callable[[Tuple[int, ...], Any], int]")