New semantic analyzer: Support more features etc. (#6268)

This adds support for several new features:

 * Nested classes
 * Built-in type aliases such as List[T]
 * Overloaded functions
 * Decorated functions

This also includes some smaller fixes related to generic classes, adds
tests for other features, and makes the order of processing module top
levels consistent across iterations.

Previously we would process top levels in alternating orders if they
are deferred (a->b, b->a). This would result in the same module (b)
processed twice in a row, which may not be efficient if the deferral
is due to another module in the cycle, as the second step may then
make no progress.

Also remove fake definitions of these aliases from stubs, as
they are not necessary.

Author: JukkaL

mypy/newsemanal/semanal.py

@@ -152,7 +152,7 @@
 
 # Special cased built-in classes that are needed for basic functionality and need to be
 # available very early on.
-CORE_BUILTIN_CLASSES = ['object', 'bool']  # type: Final
+CORE_BUILTIN_CLASSES = ['object', 'bool', 'tuple']  # type: Final
 
 
 # Used for tracking incomplete references
@@ -390,6 +390,8 @@ def refresh_top_level(self, file_node: MypyFile) -> None:
         self.recurse_into_functions = False
         for d in file_node.defs:
             self.accept(d)
+        if file_node.fullname() == 'typing':
+            self.add_builtin_aliases(file_node)
 
     @contextmanager
     def file_context(self, file_node: MypyFile, fnam: str, options: Options,
@@ -432,15 +434,19 @@ def file_context(self, file_node: MypyFile, fnam: str, options: Options,
         del self.options
 
     def visit_func_def(self, defn: FuncDef) -> None:
-        self.add_func_to_symbol_table(defn)
+        if not defn.is_decorated and not defn.is_overload:
+            self.add_func_to_symbol_table(defn)
 
         if not self.recurse_into_functions:
             return
 
         with self.scope.function_scope(defn):
             self._visit_func_def(defn)
 
-    def add_func_to_symbol_table(self, func: FuncDef) -> None:
+    def add_func_to_symbol_table(self, func: Union[FuncDef, OverloadedFuncDef]) -> None:
+        if self.is_class_scope():
+            assert self.type is not None
+            func.info = self.type
         func._fullname = self.qualified_name(func.name())
         self.add_symbol(func.name(), func, func)
 
@@ -548,8 +554,11 @@ def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem)
             fun_type.variables = a.bind_function_type_variables(fun_type, defn)
 
     def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:
+        self.add_func_to_symbol_table(defn)
+
         if not self.recurse_into_functions:
             return
+
         # NB: Since _visit_overloaded_func_def will call accept on the
         # underlying FuncDefs, the function might get entered twice.
         # This is fine, though, because only the outermost function is
@@ -580,7 +589,7 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:
             # This is an a normal overload. Find the item signatures, the
             # implementation (if outside a stub), and any missing @overload
             # decorators.
-            types, impl, non_overload_indexes = self.find_overload_sigs_and_impl(defn)
+            types, impl, non_overload_indexes = self.analyze_overload_sigs_and_impl(defn)
             defn.impl = impl
             if non_overload_indexes:
                 self.handle_missing_overload_decorators(defn, non_overload_indexes,
@@ -606,15 +615,9 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:
         self.process_final_in_overload(defn)
         self.process_static_or_class_method_in_overload(defn)
 
-        # TODO: Use add_symbol or similar below (this is inconsistent)
-        if self.is_class_scope():
-            assert self.type is not None
-            self.type.names[defn.name()] = SymbolTableNode(MDEF, defn)
-            defn.info = self.type
-        elif self.is_func_scope():
-            self.add_local(defn, defn)
+        self.add_symbol(defn.name(), defn, defn)
 
-    def find_overload_sigs_and_impl(
+    def analyze_overload_sigs_and_impl(
             self,
             defn: OverloadedFuncDef) -> Tuple[List[CallableType],
                                               Optional[OverloadPart],
@@ -1165,10 +1168,11 @@ def analyze_class_typevar_declaration(self, base: Type) -> Optional[TypeVarList]
                 sym.node.fullname() == 'typing_extensions.Protocol' and base.args):
             tvars = []  # type: TypeVarList
             for arg in unbound.args:
+                tag = self.track_incomplete_refs()
                 tvar = self.analyze_unbound_tvar(arg)
                 if tvar:
                     tvars.append(tvar)
-                else:
+                elif not self.found_incomplete_ref(tag):
                     self.fail('Free type variable expected in %s[...]' %
                               sym.node.name(), base)
             return tvars
@@ -1216,7 +1220,7 @@ def prepare_class_def(self, defn: ClassDef, info: Optional[TypeInfo] = None) ->
             info = info or self.make_empty_type_info(defn)
             defn.info = info
             info.defn = defn
-            if self.is_module_scope():
+            if not self.is_func_scope():
                 info._fullname = self.qualified_name(defn.name)
             else:
                 info._fullname = info.name()
@@ -2757,6 +2761,9 @@ def process_module_assignment(self, lvals: List[Lvalue], rval: Expression,
                             lnode.node = rnode.node
 
     def visit_decorator(self, dec: Decorator) -> None:
+        if not dec.is_overload:
+            self.add_symbol(dec.name(), dec, dec)
+        dec.func._fullname = self.qualified_name(dec.name())
         for d in dec.decorators:
             d.accept(self)
         removed = []  # type: List[int]
@@ -3362,7 +3369,9 @@ def visit_index_expr(self, expr: IndexExpr) -> None:
                 # may be analysing a type alias definition rvalue. The error will be
                 # reported elsewhere if it is not the case.
                 typearg2 = self.anal_type(typearg, allow_unbound_tvars=True)
-                assert typearg2 is not None  # TODO: Deal with None return values
+                if typearg2 is None:
+                    self.defer()
+                    return
                 types.append(typearg2)
             expr.analyzed = TypeApplication(expr.base, types)
             expr.analyzed.line = expr.line
@@ -3726,17 +3735,23 @@ def add_builtin_aliases(self, tree: MypyFile) -> None:
         assert tree.fullname() == 'typing'
         for alias, target_name in type_aliases.items():
             name = alias.split('.')[-1]
+            if name in tree.names:
+                continue
+            tag = self.track_incomplete_refs()
             n = self.lookup_fully_qualified_or_none(target_name)
             if n:
+                # Found built-in class target. Create alias.
                 target = self.named_type_or_none(target_name, [])
                 assert target is not None
                 alias_node = TypeAlias(target, alias, line=-1, column=-1,  # there is no context
                                        no_args=True, normalized=True)
-                tree.names[name] = SymbolTableNode(GDEF, alias_node)
+                self.add_symbol(name, alias_node, tree)
+            elif self.found_incomplete_ref(tag):
+                # Built-in class target may not ready yet -- defer.
+                self.mark_incomplete(name)
             else:
-                # Built-in target not defined, remove the original fake
-                # definition to trigger a better error message.
-                tree.names.pop(name, None)
+                # Test fixtures may be missing some builtin classes, which is okay.
+                pass
 
     def lookup_fully_qualified(self, name: str) -> SymbolTableNode:
         """Lookup a fully qualified name.
@@ -3769,7 +3784,11 @@ def lookup_fully_qualified_or_none(self, fullname: str) -> Optional[SymbolTableN
         if module not in self.modules:
             return None
         filenode = self.modules[module]
-        return filenode.names.get(name)
+        result = filenode.names.get(name)
+        if result is None and self.is_incomplete_namespace(module):
+            # TODO: More explicit handling of incomplete refs?
+            self.record_incomplete_ref()
+        return result
 
     def qualified_name(self, n: str) -> str:
         if self.type is not None:

mypy/newsemanal/semanal_main.py

@@ -80,7 +80,9 @@ def process_top_levels(graph: 'Graph', scc: List[str]) -> None:
             all_deferred += deferred
             if not incomplete:
                 state.manager.incomplete_namespaces.discard(next_id)
-        worklist = all_deferred
+        # Reverse to process the targets in the same order on every iteration. This avoids
+        # processing the same target twice in a row, which is inefficient.
+        worklist = list(reversed(all_deferred))
 
 
 def process_functions(graph: 'Graph', scc: List[str]) -> None:
@@ -99,7 +101,7 @@ def process_functions(graph: 'Graph', scc: List[str]) -> None:
             assert not incomplete  # Ditto
 
 
-TargetInfo = Tuple[str, Union[MypyFile, FuncDef], Optional[TypeInfo]]
+TargetInfo = Tuple[str, Union[MypyFile, FuncDef, OverloadedFuncDef, Decorator], Optional[TypeInfo]]
 
 
 def get_all_leaf_targets(symtable: SymbolTable,
@@ -109,9 +111,7 @@ def get_all_leaf_targets(symtable: SymbolTable,
     result = []  # type: List[TargetInfo]
     for name, node in symtable.items():
         new_prefix = prefix + '.' + name
-        # TODO: Decorated function
-        # TODO: Overloaded function
-        if isinstance(node.node, (FuncDef, TypeInfo)):
+        if isinstance(node.node, (FuncDef, TypeInfo, OverloadedFuncDef, Decorator)):
             if node.node.fullname() == new_prefix:
                 if isinstance(node.node, TypeInfo):
                     result += get_all_leaf_targets(node.node.names, new_prefix, node.node)
@@ -122,7 +122,7 @@ def get_all_leaf_targets(symtable: SymbolTable,
 
 def semantic_analyze_target(target: str,
                             state: 'State',
-                            node: Union[MypyFile, FuncDef],
+                            node: Union[MypyFile, FuncDef, OverloadedFuncDef, Decorator],
                             active_type: Optional[TypeInfo]) -> Tuple[List[str], bool]:
     # TODO: Support refreshing function targets (currently only works for module top levels)
     tree = state.tree
@@ -137,6 +137,9 @@ def semantic_analyze_target(target: str,
                                    fnam=tree.path,
                                    options=state.options,
                                    active_type=active_type):
+            if isinstance(node, Decorator):
+                # Decorator expressions will be processed as part of the module top level.
+                node = node.func
             analyzer.refresh_partial(node, [])
     if analyzer.deferred:
         return [target], analyzer.incomplete

mypy/newsemanal/typeanal.py

@@ -334,6 +334,9 @@ def analyze_unbound_type_with_type_info(self, t: UnboundType, info: TypeInfo) ->
         # valid count at this point. Thus we may construct an
         # Instance with an invalid number of type arguments.
         instance = Instance(info, self.anal_array(t.args), t.line, t.column)
+        # Check type argument count.
+        if len(instance.args) != len(info.type_vars):
+            fix_instance(instance, self.fail)
         if not t.args and self.options.disallow_any_generics and not self.defining_alias:
             # We report/patch invalid built-in instances already during second pass.
             # This is done to avoid storing additional state on instances.