New semantic analyzer: Support builtin typing aliases (#6358)

Fixes #6297

This adds support for real typeshed stubs that define dummy aliases like `typing.List`, `typing.Dict`, etc. This also fixes couple related issues, so that builtin SCC is almost clean (the two remaining errors are #6295 and #6357).

Most notably, this PR introduces some re-ordering of targets in builtin SCC, removing this reordering requires some non-trivial work (namely #6356, #6355, and deferring targets from `named_type()`).

Author: ilevkivskyi

mypy/build.py

@@ -2584,6 +2584,11 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
         if 'builtins' in ascc:
             scc.remove('builtins')
             scc.append('builtins')
+        # HACK: similar is needed for 'typing', for untangling the builtins SCC when new semantic
+        # analyzer is used.
+        if 'typing' in ascc:
+            scc.remove('typing')
+            scc.insert(0, 'typing')
         if manager.options.verbosity >= 2:
             for id in scc:
                 manager.trace("Priorities for %s:" % id,

mypy/newsemanal/semanal.py

@@ -348,6 +348,20 @@ def prepare_file(self, file_node: MypyFile) -> None:
                                                               self.modules['builtins'])
         if file_node.fullname() == 'builtins':
             self.prepare_builtins_namespace(file_node)
+        if file_node.fullname() == 'typing':
+            self.prepare_typing_namespace(file_node)
+
+    def prepare_typing_namespace(self, file_node: MypyFile) -> None:
+        """Remove dummy alias definitions such as List = TypeAlias(object) from typing.
+
+        They will be replaced with real aliases when corresponding targets are ready.
+        """
+        for stmt in file_node.defs.copy():
+            if (isinstance(stmt, AssignmentStmt) and len(stmt.lvalues) == 1 and
+                    isinstance(stmt.lvalues[0], NameExpr)):
+                # Assignment to a simple name, remove it if it is a dummy alias.
+                if 'typing.' + stmt.lvalues[0].name in type_aliases:
+                    file_node.defs.remove(stmt)
 
     def prepare_builtins_namespace(self, file_node: MypyFile) -> None:
         names = file_node.names
@@ -2531,10 +2545,17 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> None:
         assert node is not None
         assert node.fullname is not None
         node.kind = self.current_symbol_kind()
-        type_var = TypeVarExpr(name, node.fullname, values, upper_bound, variance)
-        type_var.line = call.line
-        call.analyzed = type_var
-        node.node = type_var
+        if isinstance(node.node, TypeVarExpr):
+            # Existing definition from previous semanal iteration, use it.
+            type_var = node.node
+            type_var.values = values
+            type_var.upper_bound = upper_bound
+            type_var.variance = variance
+        else:
+            type_var = TypeVarExpr(name, node.fullname, values, upper_bound, variance)
+            type_var.line = call.line
+            call.analyzed = type_var
+            node.node = type_var
 
     def check_typevar_name(self, call: CallExpr, name: str, context: Context) -> bool:
         name = unmangle(name)

mypy/newsemanal/semanal_main.py

@@ -39,6 +39,10 @@
 # Perform up to this many semantic analysis iterations until giving up trying to bind all names.
 MAX_ITERATIONS = 10
 
+# Number of passes over core modules before going on to the rest of the builtin SCC.
+CORE_WARMUP = 2
+core_modules = ['typing', 'builtins', 'abc', 'collections']
+
 
 def semantic_analysis_for_scc(graph: 'Graph', scc: List[str]) -> None:
     """Perform semantic analysis for all modules in a SCC (import cycle).
@@ -65,6 +69,10 @@ def process_top_levels(graph: 'Graph', scc: List[str]) -> None:
     state.manager.incomplete_namespaces.update(scc)
 
     worklist = scc[:]
+    # HACK: process core stuff first. This is mostly needed to support defining
+    # named tuples in builtin SCC.
+    if all(m in worklist for m in core_modules):
+        worklist += list(reversed(core_modules)) * CORE_WARMUP
     iteration = 0
     final_iteration = False
     while worklist:

mypy/newsemanal/semanal_shared.py

@@ -42,6 +42,10 @@ def lookup_qualified(self, name: str, ctx: Context,
     def lookup_fully_qualified(self, name: str) -> SymbolTableNode:
         raise NotImplementedError
 
+    @abstractmethod
+    def lookup_fully_qualified_or_none(self, name: str) -> Optional[SymbolTableNode]:
+        raise NotImplementedError
+
     @abstractmethod
     def fail(self, msg: str, ctx: Context, serious: bool = False, *,
              blocker: bool = False) -> None:
@@ -64,6 +68,11 @@ def record_incomplete_ref(self) -> None:
     def defer(self) -> None:
         raise NotImplementedError
 
+    @abstractmethod
+    def is_incomplete_namespace(self, fullname: str) -> bool:
+        """Is a module or class namespace potentially missing some definitions?"""
+        raise NotImplementedError
+
 
 @trait
 class SemanticAnalyzerInterface(SemanticAnalyzerCoreInterface):

mypy/newsemanal/typeanal.py

@@ -285,6 +285,15 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Opt
                       " in a variable annotation", t)
             return AnyType(TypeOfAny.from_error)
         elif fullname == 'typing.Tuple':
+            # Tuple is special because it is involved in builtin import cycle
+            # and may be not ready when used.
+            sym = self.api.lookup_fully_qualified_or_none('builtins.tuple')
+            if not sym or isinstance(sym.node, PlaceholderNode):
+                if self.api.is_incomplete_namespace('builtins'):
+                    self.api.record_incomplete_ref()
+                else:
+                    self.fail("Name 'tuple' is not defined", t)
+                return AnyType(TypeOfAny.special_form)
             if len(t.args) == 0 and not t.empty_tuple_index:
                 # Bare 'Tuple' is same as 'tuple'
                 if self.options.disallow_any_generics and not self.is_typeshed_stub:
@@ -350,7 +359,8 @@ def analyze_type_with_type_info(self, info: TypeInfo, args: List[Type], ctx: Con
         # Instance with an invalid number of type arguments.
         instance = Instance(info, self.anal_array(args), ctx.line, ctx.column)
         # Check type argument count.
-        if len(instance.args) != len(info.type_vars):
+        # TODO: remove this from here and replace with a proper separate pass.
+        if len(instance.args) != len(info.type_vars) and not self.defining_alias:
             fix_instance(instance, self.fail)
         if not args and self.options.disallow_any_generics and not self.defining_alias:
             # We report/patch invalid built-in instances already during second pass.

mypy/test/hacks.py

@@ -26,7 +26,6 @@
     'check-flags.test',
     'check-functions.test',
     'check-generics.test',
-    'check-ignore.test',
     'check-incomplete-fixture.test',
     'check-incremental.test',
     'check-inference-context.test',

mypy/test/testpythoneval.py

@@ -56,6 +56,8 @@ def test_python_evaluation(testcase: DataDrivenTestCase, cache_dir: str) -> None
         '--no-strict-optional',
         '--no-silence-site-packages',
     ]
+    if testcase.name.lower().endswith('_newsemanal'):
+        mypy_cmdline.append('--new-semantic-analyzer')
     py2 = testcase.name.lower().endswith('python2')
     if py2:
         mypy_cmdline.append('--py2')
@@ -92,6 +94,9 @@ def test_python_evaluation(testcase: DataDrivenTestCase, cache_dir: str) -> None
         output.extend(interp_out)
     # Remove temp file.
     os.remove(program_path)
+    for i, line in enumerate(output):
+        if os.path.sep + 'typeshed' + os.path.sep in line:
+            output[i] = line.split(os.path.sep)[-1]
     assert_string_arrays_equal(adapt_output(testcase), output,
                                'Invalid output ({}, line {})'.format(
                                    testcase.file, testcase.line))

test-data/unit/check-newsemanal.test

@@ -899,6 +899,12 @@ class C: pass
 import a
 from b import C
 
+[case testNewAnalyzerIncompleteFixture]
+from typing import Tuple
+
+x: Tuple[int]  # E: Name 'tuple' is not defined
+[builtins fixtures/complex.pyi]
+
 [case testNewAnalyzerMetaclass1]
 class A(metaclass=B):
     pass

test-data/unit/fixtures/complex.pyi

@@ -1,4 +1,5 @@
 # Builtins stub used for some float/complex test cases.
+# Please don't add tuple to this file, it is used to test incomplete fixtures.
 
 class object:
     def __init__(self): pass

test-data/unit/pythoneval.test

@@ -1358,3 +1358,14 @@ x = X(a=1, b='s')
 
 [out]
 _testNamedTupleNew.py:12: error: Revealed type is 'Tuple[builtins.int, fallback=_testNamedTupleNew.Child]'
+
+[case testNewAnalyzerBasicTypeshed_newsemanal]
+from typing import Dict, List, Tuple
+
+x: Dict[str, List[int]]
+reveal_type(x['test'][0])
+[out]
+typing.pyi: error: Class typing.Sequence has abstract attributes "__len__"
+typing.pyi: note: If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass
+builtins.pyi:39: error: Name '__class__' already defined on line 39
+_testNewAnalyzerBasicTypeshed_newsemanal.py:4: error: Revealed type is 'builtins.int*'