Fine-grained: Fix AST merge issues (#4652)

This fixes all but one AST merge checker test failures. The remaining
one (testFineGrainedCallable) seems tricky to fix so I'll leave it for later.

I decided not to add tests for things that can be caught using the
merge checker. We'll have to run the merge checker periodically
since it's not enabled by default in the test suite.

Author: JukkaL

mypy/build.py

@@ -1992,7 +1992,9 @@ def finish_passes(self) -> None:
             return
         with self.wrap_context():
             # Some tests want to look at the set of all types.
-            if manager.options.use_builtins_fixtures or manager.options.dump_deps:
+            options = manager.options
+            if ((options.use_builtins_fixtures and not options.fine_grained_incremental) or
+                    manager.options.dump_deps):
                 manager.all_types.update(self.type_map())
 
             if self.options.incremental:

mypy/nodes.py

@@ -2206,11 +2206,18 @@ def type_str(typ: 'mypy.types.Type') -> str:
             if isinstance(node, Var) and node.type:
                 description += ' ({})'.format(type_str(node.type))
             names.append(description)
+        items = [
+            'Name({})'.format(self.fullname()),
+            base,
+            mro,
+            ('Names', names),
+        ]
+        if self.declared_metaclass:
+            items.append('DeclaredMetaclass({})'.format(type_str(self.declared_metaclass)))
+        if self.metaclass_type:
+            items.append('MetaclassType({})'.format(type_str(self.metaclass_type)))
         return mypy.strconv.dump_tagged(
-            ['Name({})'.format(self.fullname()),
-             base,
-             mro,
-             ('Names', names)],
+            items,
             head,
             str_conv=str_conv)
 

mypy/semanal.py

@@ -306,6 +306,8 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
 
             del self.options
             del self.patches
+            del self.cur_mod_node
+            del self.globals
 
     def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef]) -> None:
         """Refresh a stale target in fine-grained incremental mode."""
@@ -324,6 +326,7 @@ def refresh_top_level(self, file_node: MypyFile) -> None:
         self.recurse_into_functions = False
         for d in file_node.defs:
             self.accept(d)
+        del self.patches
 
     @contextmanager
     def file_context(self, file_node: MypyFile, fnam: str, options: Options,

mypy/semanal_pass3.py

@@ -68,6 +68,8 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
             self.scope.enter_file(file_node.fullname())
             self.accept(file_node)
             self.scope.leave()
+        del self.cur_mod_node
+        self.patches = []
 
     def refresh_partial(self, node: Union[MypyFile, FuncItem, OverloadedFuncDef]) -> None:
         """Refresh a stale target in fine-grained incremental mode."""

mypy/server/astmerge.py

@@ -50,13 +50,14 @@
 from mypy.nodes import (
     Node, MypyFile, SymbolTable, Block, AssignmentStmt, NameExpr, MemberExpr, RefExpr, TypeInfo,
     FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, SuperExpr, NewTypeExpr,
-    OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, MDEF
+    OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, FuncBase, TypeAliasExpr, CallExpr,
+    MDEF
 )
 from mypy.traverser import TraverserVisitor
 from mypy.types import (
     Type, TypeVisitor, Instance, AnyType, NoneTyp, CallableType, DeletedType, PartialType,
     TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType,
-    Overloaded, TypeVarDef
+    Overloaded, TypeVarDef, TypeList
 )
 from mypy.util import get_prefix
 
@@ -156,34 +157,41 @@ def visit_block(self, node: Block) -> None:
 
     def visit_func_def(self, node: FuncDef) -> None:
         node = self.fixup(node)
-        if node.type:
-            self.fixup_type(node.type)
-        if node.info:
-            node.info = self.fixup(node.info)
+        self.process_base_func(node)
         super().visit_func_def(node)
 
     def visit_overloaded_func_def(self, node: OverloadedFuncDef) -> None:
-        if node.info:
-            node.info = self.fixup(node.info)
+        self.process_base_func(node)
         super().visit_overloaded_func_def(node)
 
     def visit_class_def(self, node: ClassDef) -> None:
         # TODO additional things?
         node.defs.body = self.replace_statements(node.defs.body)
         node.info = self.fixup(node.info)
+        info = node.info
         for tv in node.type_vars:
             self.process_type_var_def(tv)
-        self.process_type_info(node.info)
+        if info:
+            if info.is_named_tuple:
+                self.process_synthetic_type_info(info)
+            else:
+                self.process_type_info(info)
         super().visit_class_def(node)
 
+    def process_base_func(self, node: FuncBase) -> None:
+        self.fixup_type(node.type)
+        node.info = self.fixup(node.info)
+        if node.unanalyzed_type:
+            # Unanalyzed types can have AST node references
+            self.fixup_type(node.unanalyzed_type)
+
     def process_type_var_def(self, tv: TypeVarDef) -> None:
         for value in tv.values:
             self.fixup_type(value)
         self.fixup_type(tv.upper_bound)
 
     def visit_assignment_stmt(self, node: AssignmentStmt) -> None:
-        if node.type:
-            self.fixup_type(node.type)
+        self.fixup_type(node.type)
         super().visit_assignment_stmt(node)
 
     # Expressions
@@ -200,46 +208,56 @@ def visit_member_expr(self, node: MemberExpr) -> None:
     def visit_ref_expr(self, node: RefExpr) -> None:
         if node.node is not None:
             node.node = self.fixup(node.node)
+            if isinstance(node.node, Var):
+                # The Var node may be an orphan and won't otherwise be processed.
+                fixup_var(node.node, self.replacements)
 
     def visit_namedtuple_expr(self, node: NamedTupleExpr) -> None:
         super().visit_namedtuple_expr(node)
         node.info = self.fixup(node.info)
-        self.process_type_info(node.info)
+        self.process_synthetic_type_info(node.info)
 
     def visit_super_expr(self, node: SuperExpr) -> None:
         super().visit_super_expr(node)
         if node.info is not None:
             node.info = self.fixup(node.info)
 
+    def visit_call_expr(self, node: CallExpr) -> None:
+        super().visit_call_expr(node)
+        if isinstance(node.analyzed, SymbolNode):
+            node.analyzed = self.fixup(node.analyzed)
+
     def visit_newtype_expr(self, node: NewTypeExpr) -> None:
         if node.info:
             node.info = self.fixup(node.info)
-            self.process_type_info(node.info)
-        if node.old_type:
-            self.fixup_type(node.old_type)
+            self.process_synthetic_type_info(node.info)
+        self.fixup_type(node.old_type)
         super().visit_newtype_expr(node)
 
     def visit_lambda_expr(self, node: LambdaExpr) -> None:
-        if node.info:
-            node.info = self.fixup(node.info)
+        node.info = self.fixup(node.info)
         super().visit_lambda_expr(node)
 
     def visit_typeddict_expr(self, node: TypedDictExpr) -> None:
-        node.info = self.fixup(node.info)
         super().visit_typeddict_expr(node)
+        node.info = self.fixup(node.info)
+        self.process_synthetic_type_info(node.info)
 
     def visit_enum_call_expr(self, node: EnumCallExpr) -> None:
         node.info = self.fixup(node.info)
-        self.process_type_info(node.info)
+        self.process_synthetic_type_info(node.info)
         super().visit_enum_call_expr(node)
 
+    def visit_type_alias_expr(self, node: TypeAliasExpr) -> None:
+        self.fixup_type(node.type)
+        self.fixup_type(node.fallback)
+        super().visit_type_alias_expr(node)
+
     # Others
 
     def visit_var(self, node: Var) -> None:
-        if node.info:
-            node.info = self.fixup(node.info)
-        if node.type:
-            self.fixup_type(node.type)
+        node.info = self.fixup(node.info)
+        self.fixup_type(node.type)
         super().visit_var(node)
 
     # Helpers
@@ -251,25 +269,34 @@ def fixup(self, node: SN) -> SN:
             return cast(SN, new)
         return node
 
-    def fixup_type(self, typ: Type) -> None:
-        typ.accept(TypeReplaceVisitor(self.replacements))
+    def fixup_type(self, typ: Optional[Type]) -> None:
+        if typ is not None:
+            typ.accept(TypeReplaceVisitor(self.replacements))
 
     def process_type_info(self, info: Optional[TypeInfo]) -> None:
         if info is None:
             return
-        # TODO: Additional things:
-        # - declared_metaclass
-        # - metaclass_type
-        # - _promote
-        # - typeddict_type
-        # - replaced
+        self.fixup_type(info.declared_metaclass)
+        self.fixup_type(info.metaclass_type)
+        self.fixup_type(info._promote)
+        self.fixup_type(info.tuple_type)
+        self.fixup_type(info.typeddict_type)
+        info.defn.info = self.fixup(info)
+        info.replaced = self.fixup(info.replaced)
         replace_nodes_in_symbol_table(info.names, self.replacements)
         for i, item in enumerate(info.mro):
             info.mro[i] = self.fixup(info.mro[i])
         for i, base in enumerate(info.bases):
             self.fixup_type(info.bases[i])
-        if info.tuple_type:
-            self.fixup_type(info.tuple_type)
+
+    def process_synthetic_type_info(self, info: TypeInfo) -> None:
+        # Synthetic types (types not created using a class statement) don't
+        # have bodies in the AST so we need to iterate over their symbol
+        # tables separately, unlike normal classes.
+        self.process_type_info(info)
+        for name, node in info.names.items():
+            if node.node:
+                node.node.accept(self)
 
     def replace_statements(self, nodes: List[Statement]) -> List[Statement]:
         result = []
@@ -345,6 +372,10 @@ def visit_unbound_type(self, typ: UnboundType) -> None:
         for arg in typ.args:
             arg.accept(self)
 
+    def visit_type_list(self, typ: TypeList) -> None:
+        for item in typ.items:
+            item.accept(self)
+
     def visit_uninhabited_type(self, typ: UninhabitedType) -> None:
         pass
 
@@ -371,15 +402,16 @@ def replace_nodes_in_symbol_table(symbols: SymbolTable,
                 new = replacements[node.node]
                 new.__dict__ = node.node.__dict__
                 node.node = new
-                # TODO: Other node types
-                if isinstance(node.node, Var) and node.node.type:
-                    node.node.type.accept(TypeReplaceVisitor(replacements))
-                    node.node.info = cast(TypeInfo, replacements.get(node.node.info,
-                                                                     node.node.info))
-            else:
-                # TODO: Other node types
-                if isinstance(node.node, Var) and node.node.type:
-                    node.node.type.accept(TypeReplaceVisitor(replacements))
+            if isinstance(node.node, Var):
+                # Handle them here just in case these aren't exposed through the AST.
+                # TODO: Is this necessary?
+                fixup_var(node.node, replacements)
         override = node.type_override
         if override:
             override.accept(TypeReplaceVisitor(replacements))
+
+
+def fixup_var(node: Var, replacements: Dict[SymbolNode, SymbolNode]) -> None:
+    if node.type:
+        node.type.accept(TypeReplaceVisitor(replacements))
+    node.info = cast(TypeInfo, replacements.get(node.info, node.info))

mypy/server/objgraph.py

@@ -23,13 +23,6 @@
     '__name__',
     '__class__',
     '__dict__',
-
-    # Mypy specific attribute blacklists
-    'indirection_detector',
-    'all_types',
-    'type_maps',
-    'semantic_analyzer',  # Semantic analyzer has stale caches
-    'semantic_analyzer_pass3',  # Semantic analyzer has stale caches
 }
 
 # Instances of these types can't have references to other objects

mypy/server/update.py

@@ -698,9 +698,11 @@ def replace_modules_with_new_variants(
     for id in new_modules:
         new_module = new_modules[id]
         if id in old_modules and new_module is not None:
-            merge_asts(old_modules[id], old_modules[id].names,
+            preserved_module = old_modules[id]
+            merge_asts(preserved_module, old_modules[id].names,
                        new_module, new_module.names)
-            manager.modules[id] = old_modules[id]
+            manager.modules[id] = preserved_module
+            graph[id].tree = preserved_module
 
 
 def propagate_changes_using_dependencies(

mypy/test/testmerge.py

@@ -8,7 +8,8 @@
 from mypy.build import BuildManager, BuildSource, State
 from mypy.errors import Errors, CompileError
 from mypy.nodes import (
-    Node, MypyFile, SymbolTable, SymbolTableNode, TypeInfo, Expression, Var, UNBOUND_IMPORTED
+    Node, MypyFile, SymbolTable, SymbolTableNode, TypeInfo, Expression, Var, TypeVarExpr,
+    UNBOUND_IMPORTED
 )
 from mypy.options import Options
 from mypy.server.astmerge import merge_asts
@@ -86,6 +87,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         a.extend(self.dump(manager, kind))
 
         for expr in old_subexpr:
+            if isinstance(expr, TypeVarExpr):
+                # These are merged so we can't perform the check.
+                continue
             # Verify that old AST nodes are removed from the expression type map.
             assert expr not in new_types
 

mypy/traverser.py

@@ -59,6 +59,8 @@ def visit_class_def(self, o: ClassDef) -> None:
         for base in o.base_type_exprs:
             base.accept(self)
         o.defs.accept(self)
+        if o.analyzed:
+            o.analyzed.accept(self)
 
     def visit_decorator(self, o: Decorator) -> None:
         o.func.accept(self)

mypy/types.py

@@ -242,9 +242,10 @@ def serialize(self) -> JsonDict:
 class TypeList(Type):
     """Information about argument types and names [...].
 
-    This is only used for the arguments of a Callable type, i.e. for
+    This is used for the arguments of a Callable type, i.e. for
     [arg, ...] in Callable[[arg, ...], ret]. This is not a real type
-    but a syntactic AST construct.
+    but a syntactic AST construct. UnboundTypes can also have TypeList
+    types before they are processed into Callable types.
     """
 
     items = None  # type: List[Type]
@@ -254,7 +255,6 @@ def __init__(self, items: List[Type], line: int = -1, column: int = -1) -> None:
         self.items = items
 
     def accept(self, visitor: 'TypeVisitor[T]') -> T:
-        assert isinstance(visitor, SyntheticTypeVisitor)
         return visitor.visit_type_list(self)
 
     def serialize(self) -> JsonDict:
@@ -1503,6 +1503,11 @@ def visit_type_type(self, t: TypeType) -> T:
     def visit_forwardref_type(self, t: ForwardRef) -> T:
         raise RuntimeError('Internal error: unresolved forward reference')
 
+    def visit_type_list(self, t: TypeList) -> T:
+        # TODO: Do we need to implement this in more visitors? TypeList objects can
+        #   exist as components of UnboundTypes.
+        raise self._notimplemented_helper('type_list')
+
 
 class SyntheticTypeVisitor(TypeVisitor[T]):
     """A TypeVisitor that also knows how to visit synthetic AST constructs.