New semantic analyzer: enable remaining tests and fix issues (#6969)

This enables incremental mode tests for the new semantic analyzer. Some
tests are skipped, mostly because of issues related to recursive type
definitions and plugins.

Also fix some issues related to type aliases and recursive definitions.

Remove support for skipping test files when using the new semantic
analyzer.

Author: JukkaL

mypy/newsemanal/semanal.py

@@ -2302,6 +2302,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
 
         Return True if it is a type alias (even if the target is not ready),
         or False otherwise.
+
         Note: the resulting types for subscripted (including generic) aliases
         are also stored in rvalue.analyzed.
         """
@@ -2320,11 +2321,13 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
         #     B = int
         #     B = float  # Error!
         # Don't create an alias in these cases:
-        if existing and (isinstance(existing.node, Var) or  # existing variable
-                isinstance(existing.node, TypeAlias) and not s.is_alias_def or  # existing alias
-                (isinstance(existing.node, PlaceholderNode) and
-                # TODO: find a more robust way to track the order of definitions.
-                 existing.node.node.line < s.line)):  # or previous incomplete definition
+        if (existing
+                and (isinstance(existing.node, Var)  # existing variable
+                     or (isinstance(existing.node, TypeAlias)
+                         and not s.is_alias_def)  # existing alias
+                     or (isinstance(existing.node, PlaceholderNode)
+                         and existing.node.node.line < s.line))):  # previous incomplete definition
+            # TODO: find a more robust way to track the order of definitions.
             # Note: if is_alias_def=True, this is just a node from previous iteration.
             if isinstance(existing.node, TypeAlias) and not s.is_alias_def:
                 self.fail('Cannot assign multiple types to name "{}"'
@@ -2349,6 +2352,10 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
         if not self.can_be_type_alias(rvalue):
             return False
 
+        if existing and not isinstance(existing.node, (PlaceholderNode, TypeAlias)):
+            # Cannot redefine existing node as type alias.
+            return False
+
         res = None  # type: Optional[Type]
         if self.is_none_alias(rvalue):
             res = NoneType()
@@ -2390,21 +2397,27 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
         alias_node = TypeAlias(res, self.qualified_name(lvalue.name), s.line, s.column,
                                alias_tvars=alias_tvars, no_args=no_args)
         if existing:
-            # Did alias get updated?
-            if (isinstance(existing.node, PlaceholderNode) or
-                    isinstance(existing.node, TypeAlias) and existing.node.target != res):
-                self.progress = True
-                # We need to defer so that this change can get propagated to base classes.
-                self.defer()
+            # An alias gets updated.
+            if self.final_iteration:
+                self.cannot_resolve_name(lvalue.name, 'name', s)
+                return True
+            updated = False
             if isinstance(existing.node, TypeAlias):
-                # Copy expansion to the existing alias, this matches how we update base classes
-                # for a TypeInfo _in place_ if there are nested placeholders.
-                existing.node.target = res
-                existing.node.alias_tvars = alias_tvars
-                existing.node.no_args = no_args
+                if existing.node.target != res:
+                    # Copy expansion to the existing alias, this matches how we update base classes
+                    # for a TypeInfo _in place_ if there are nested placeholders.
+                    existing.node.target = res
+                    existing.node.alias_tvars = alias_tvars
+                    existing.node.no_args = no_args
+                    updated = True
             else:
                 # Otherwise just replace existing placeholder with type alias.
                 existing.node = alias_node
+                updated = True
+            if updated:
+                self.progress = True
+                # We need to defer so that this change can get propagated to base classes.
+                self.defer()
         else:
             self.add_symbol(lvalue.name, alias_node, s)
         if isinstance(rvalue, RefExpr) and isinstance(rvalue.node, TypeAlias):

mypy/newsemanal/typeanal.py

@@ -165,22 +165,21 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool)
                 if node.becomes_typeinfo:
                     # Reference to placeholder type.
                     if self.api.final_iteration:
-                        # TODO: Move error message generation to messages.py. We'd first
-                        #       need access to MessageBuilder here. Also move the similar
-                        #       message generation logic in semanal.py.
-                        self.api.fail(
-                            'Cannot resolve name "{}" (possible cyclic definition)'.format(t.name),
-                            t)
+                        self.cannot_resolve_type(t)
                         return AnyType(TypeOfAny.from_error)
                     elif self.allow_placeholder:
                         self.api.defer()
                     else:
                         self.api.record_incomplete_ref()
                     return PlaceholderType(node.fullname(), self.anal_array(t.args), t.line)
                 else:
-                    # Reference to an unknown placeholder node.
-                    self.api.record_incomplete_ref()
-                    return AnyType(TypeOfAny.special_form)
+                    if self.api.final_iteration:
+                        self.cannot_resolve_type(t)
+                        return AnyType(TypeOfAny.from_error)
+                    else:
+                        # Reference to an unknown placeholder node.
+                        self.api.record_incomplete_ref()
+                        return AnyType(TypeOfAny.special_form)
             if node is None:
                 self.fail('Internal error (node is None, kind={})'.format(sym.kind), t)
                 return AnyType(TypeOfAny.special_form)
@@ -224,6 +223,14 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool)
         else:  # sym is None
             return AnyType(TypeOfAny.special_form)
 
+    def cannot_resolve_type(self, t: UnboundType) -> None:
+        # TODO: Move error message generation to messages.py. We'd first
+        #       need access to MessageBuilder here. Also move the similar
+        #       message generation logic in semanal.py.
+        self.api.fail(
+            'Cannot resolve name "{}" (possible cyclic definition)'.format(t.name),
+            t)
+
     def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Optional[Type]:
         """Bind special type that is recognized through magic name such as 'typing.Any'.
 

mypy/test/data.py

@@ -14,7 +14,6 @@
 from typing import List, Tuple, Set, Optional, Iterator, Any, Dict, NamedTuple, Union
 
 from mypy.test.config import test_data_prefix, test_temp_dir, PREFIX
-from mypy.test.hacks import new_semanal_blacklist
 
 root_dir = os.path.normpath(PREFIX)
 
@@ -205,8 +204,6 @@ def __init__(self,
         if ((platform == 'windows' and sys.platform != 'win32')
                 or (platform == 'posix' and sys.platform == 'win32')):
             skip = True
-        if os.getenv('NEWSEMANAL') and os.path.split(file)[-1] in new_semanal_blacklist:
-            skip = True
         self.skip = skip
         self.data = data
         self.line = line

mypy/test/hacks.py

@@ -2465,6 +2465,7 @@ x: int = C()[0][0]
 [out]
 
 [case testForwardTypeAliasInBase2]
+# flags: --new-semantic-analyzer
 from typing import List, Generic, TypeVar, NamedTuple
 T = TypeVar('T')
 
@@ -2475,12 +2476,13 @@ A = G[C]
 class B(NamedTuple):
     x: int
 
-C().x
-C()[0]
+C(1).x
+C(1)[0]
 [builtins fixtures/list.pyi]
 [out]
 
 [case testSerializeRecursiveAliases1]
+# flags: --no-new-semantic-analyzer
 from typing import Type, Callable, Union
 
 A = Union[A, int]  # type: ignore
@@ -2489,6 +2491,7 @@ C = Type[C] # type: ignore
 [out]
 
 [case testSerializeRecursiveAliases2]
+# flags: --no-new-semantic-analyzer
 from typing import Type, Callable, Union
 
 A = Union[B, int]  # type: ignore
@@ -2497,6 +2500,7 @@ C = Type[A] # type: ignore
 [out]
 
 [case testSerializeRecursiveAliases3]
+# flags: --no-new-semantic-analyzer
 from typing import Type, Callable, Union, NamedTuple
 
 A = Union[B, int]  # type: ignore
@@ -2566,6 +2570,7 @@ M = NamedTuple('M', [('x', int)])
 [out]
 
 [case testSelfRefNTIncremental1]
+# flags: --no-new-semantic-analyzer
 from typing import Tuple, NamedTuple
 
 Node = NamedTuple('Node', [ # type: ignore
@@ -2576,6 +2581,7 @@ n: Node
 [builtins fixtures/tuple.pyi]
 
 [case testSelfRefNTIncremental2]
+# flags: --no-new-semantic-analyzer
 from typing import Tuple, NamedTuple
 
 A = NamedTuple('A', [ # type: ignore
@@ -2590,6 +2596,7 @@ n: A
 [builtins fixtures/tuple.pyi]
 
 [case testSelfRefNTIncremental3]
+# flags: --no-new-semantic-analyzer
 from typing import NamedTuple, Tuple
 
 class B(NamedTuple): # type: ignore
@@ -2605,6 +2612,7 @@ lst = [m, n]
 [builtins fixtures/tuple.pyi]
 
 [case testSelfRefNTIncremental4]
+# flags: --no-new-semantic-analyzer
 from typing import NamedTuple
 
 class B(NamedTuple): # type: ignore
@@ -2618,6 +2626,7 @@ n: A
 [builtins fixtures/tuple.pyi]
 
 [case testSelfRefNTIncremental5]
+# flags: --no-new-semantic-analyzer
 from typing import NamedTuple
 
 B = NamedTuple('B', [ # type: ignore
@@ -3272,6 +3281,7 @@ main:8: error: Revealed type is 'def (x: builtins.str) -> __main__.A@6'
 main:8: error: Revealed type is 'def (x: builtins.str) -> __main__.A@6'
 
 [case testAttrsIncrementalConverterInSubmoduleForwardRef]
+# flags: --no-new-semantic-analyzer
 from a.a import A
 reveal_type(A)
 [file a/__init__.py]
@@ -3289,11 +3299,12 @@ F = List[int]
 
 [builtins fixtures/list.pyi]
 [out1]
-main:2: error: Revealed type is 'def (x: builtins.list[builtins.int]) -> a.a.A'
+main:3: error: Revealed type is 'def (x: builtins.list[builtins.int]) -> a.a.A'
 [out2]
-main:2: error: Revealed type is 'def (x: builtins.list[builtins.int]) -> a.a.A'
+main:3: error: Revealed type is 'def (x: builtins.list[builtins.int]) -> a.a.A'
 
 [case testAttrsIncrementalConverterType]
+# flags: --no-new-semantic-analyzer
 from a import C
 import attr
 o = C("1", "2", "3", "4")
@@ -3324,11 +3335,11 @@ class C:
     d: int = attr.ib(converter=parse)
 [builtins fixtures/attr.pyi]
 [out1]
-main:5: error: Revealed type is 'def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str]) -> a.C'
-main:9: error: Revealed type is 'def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str], x: builtins.str) -> __main__.D'
+main:6: error: Revealed type is 'def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str]) -> a.C'
+main:10: error: Revealed type is 'def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str], x: builtins.str) -> __main__.D'
 [out2]
-main:5: error: Revealed type is 'def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str]) -> a.C'
-main:9: error: Revealed type is 'def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str], x: builtins.str) -> __main__.D'
+main:6: error: Revealed type is 'def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str]) -> a.C'
+main:10: error: Revealed type is 'def (a: Union[builtins.float, builtins.str], b: Union[builtins.str, builtins.bytes, builtins.int], c: builtins.str, d: Union[builtins.int, builtins.str], x: builtins.str) -> __main__.D'
 
 [case testAttrsIncrementalThreeRuns]
 from a import A
@@ -4461,6 +4472,7 @@ def outer() -> None:
 [out2]
 
 [case testRecursiveAliasImported]
+# flags: --no-new-semantic-analyzer
 import a
 [file a.py]
 import lib
@@ -4482,7 +4494,38 @@ B = List[A]
 [out2]
 tmp/a.py:3: error: Revealed type is 'builtins.list[builtins.list[builtins.list[Any]]]'
 
+[case testRecursiveAliasImported2]
+# flags: --new-semantic-analyzer
+import a
+[file a.py]
+import lib
+x: int
+[file a.py.2]
+import lib
+x: lib.A
+reveal_type(x)
+[file lib.pyi]
+from typing import List
+from other import B
+A = List[B]  # type: ignore
+[file other.pyi]
+from typing import List
+from lib import A
+B = List[A]
+[builtins fixtures/list.pyi]
+[out]
+tmp/other.pyi:2: error: Module 'lib' has no attribute 'A'
+tmp/other.pyi:3: error: Cannot resolve name "B" (possible cyclic definition)
+tmp/lib.pyi:2: error: Module 'other' has no attribute 'B'
+[out2]
+tmp/other.pyi:2: error: Module 'lib' has no attribute 'A'
+tmp/other.pyi:3: error: Cannot resolve name "B" (possible cyclic definition)
+tmp/lib.pyi:2: error: Module 'other' has no attribute 'B'
+tmp/a.py:2: error: Cannot resolve name "lib.A" (possible cyclic definition)
+tmp/a.py:3: error: Revealed type is 'Any'
+
 [case testRecursiveNamedTupleTypedDict]
+# flags: --no-new-semantic-analyzer
 import a
 [file a.py]
 import lib
@@ -4576,6 +4619,8 @@ def f(x: str) -> None: pass
 tmp/main.py:2: error: Argument 1 to "f" has incompatible type "int"; expected "str"
 
 [case testOverrideByIdemAlias]
+# https://github.com/python/mypy/issues/6404
+# flags: --no-new-semantic-analyzer
 import a
 [file a.py]
 import lib

test-data/unit/check-incremental.test

@@ -2446,3 +2446,18 @@ def dec(f): pass
 # 5 padding
 # 6 padding
 def f(): pass
+
+[case testNewAnalyzerOverrideClassWithTypeAlias]
+from typing import Generic, TypeVar
+
+T = TypeVar('T')
+
+class C(Generic[T]):
+    pass
+# TODO: Error message is confusing
+C = C[int]  # E: Cannot assign to a type \
+            # E: Incompatible types in assignment (expression has type "Type[__main__.C[Any]]", variable has type "Type[__main__.C[Any]]")
+x: C
+reveal_type(x) # E: Revealed type is '__main__.C[Any]'
+[out]
+[out2]