New semantic analyzer: abstract classes (#6423)

This fixes abstract classes by moving determination of abstract
attributes to a new pass that happens between the main semantic
analysis pass and type checking. The new pass is added mostly
to simplify the main semantic analysis pass.

This also adds simple type inference for decorated functions from
the old semantic analysis pass 3. This fixes a test case where
we need access to the signature of an abstract method when type
checking the module top level. Since module top levels can't be
deferred during type checking, the simple type inference is needed
to remain compatible with the old semantic analyzer.

Fixes #6322.

Author: JukkaL

mypy/build.py

@@ -2833,7 +2833,7 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No
         if not manager.options.new_semantic_analyzer:
             manager.semantic_analyzer.add_builtin_aliases(typing_mod)
     if manager.options.new_semantic_analyzer:
-        semantic_analysis_for_scc(graph, scc)
+        semantic_analysis_for_scc(graph, scc, manager.errors)
     else:
         for id in stale:
             graph[id].semantic_analysis()

mypy/newsemanal/semanal.py

@@ -937,7 +937,6 @@ def analyze_class_body_common(self, defn: ClassDef) -> None:
         """Parts of class body analysis that are common to all kinds of class defs."""
         self.enter_class(defn.info)
         defn.defs.accept(self)
-        self.calculate_abstract_status(defn.info)
         self.apply_class_plugin_hooks(defn)
         self.leave_class()
 
@@ -1028,56 +1027,6 @@ def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None
                                         'typing_extensions.final'):
                 defn.info.is_final = True
 
-    def calculate_abstract_status(self, typ: TypeInfo) -> None:
-        """Calculate abstract status of a class.
-
-        Set is_abstract of the type to True if the type has an unimplemented
-        abstract attribute.  Also compute a list of abstract attributes.
-        """
-        concrete = set()  # type: Set[str]
-        abstract = []  # type: List[str]
-        abstract_in_this_class = []  # type: List[str]
-        for base in typ.mro:
-            for name, symnode in base.names.items():
-                node = symnode.node
-                if isinstance(node, OverloadedFuncDef):
-                    # Unwrap an overloaded function definition. We can just
-                    # check arbitrarily the first overload item. If the
-                    # different items have a different abstract status, there
-                    # should be an error reported elsewhere.
-                    func = node.items[0]  # type: Optional[Node]
-                else:
-                    func = node
-                if isinstance(func, Decorator):
-                    fdef = func.func
-                    if fdef.is_abstract and name not in concrete:
-                        typ.is_abstract = True
-                        abstract.append(name)
-                        if base is typ:
-                            abstract_in_this_class.append(name)
-                elif isinstance(node, Var):
-                    if node.is_abstract_var and name not in concrete:
-                        typ.is_abstract = True
-                        abstract.append(name)
-                        if base is typ:
-                            abstract_in_this_class.append(name)
-                concrete.add(name)
-        # In stubs, abstract classes need to be explicitly marked because it is too
-        # easy to accidentally leave a concrete class abstract by forgetting to
-        # implement some methods.
-        typ.abstract_attributes = sorted(abstract)
-        if not self.is_stub_file:
-            return
-        if (typ.declared_metaclass and typ.declared_metaclass.type.fullname() == 'abc.ABCMeta'):
-            return
-        if typ.is_protocol:
-            return
-        if abstract and not abstract_in_this_class:
-            attrs = ", ".join('"{}"'.format(attr) for attr in sorted(abstract))
-            self.fail("Class {} has abstract attributes {}".format(typ.fullname(), attrs), typ)
-            self.note("If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass",
-                      typ)
-
     def setup_type_promotion(self, defn: ClassDef) -> None:
         """Setup extra, ad-hoc subtyping relationships between classes (promotion).
 

mypy/newsemanal/semanal_abstract.py

@@ -0,0 +1,80 @@
+"""Calculate the abstract status of classes.
+
+This happens after semantic analysis and before type checking.
+"""
+
+from typing import List, Set, Optional
+
+from mypy.nodes import Node, MypyFile, SymbolTable, TypeInfo, Var, Decorator, OverloadedFuncDef
+from mypy.errors import Errors
+
+
+def calculate_abstract_status(file: MypyFile, errors: Errors) -> None:
+    """Calculate the abstract status of all classes in the symbol table in file.
+
+    Also check that ABCMeta is used correctly.
+    """
+    process(file.names, file.is_stub, file.fullname(), errors)
+
+
+def process(names: SymbolTable, is_stub_file: bool, prefix: str, errors: Errors) -> None:
+    for name, symnode in names.items():
+        node = symnode.node
+        if isinstance(node, TypeInfo) and node.fullname().startswith(prefix):
+            calculate_class_abstract_status(node, is_stub_file, errors)
+            new_prefix = prefix + '.' + node.name()
+            process(node.names, is_stub_file, new_prefix, errors)
+
+
+def calculate_class_abstract_status(typ: TypeInfo, is_stub_file: bool, errors: Errors) -> None:
+    """Calculate abstract status of a class.
+
+    Set is_abstract of the type to True if the type has an unimplemented
+    abstract attribute.  Also compute a list of abstract attributes.
+    Report error is required ABCMeta metaclass is missing.
+    """
+    concrete = set()  # type: Set[str]
+    abstract = []  # type: List[str]
+    abstract_in_this_class = []  # type: List[str]
+    for base in typ.mro:
+        for name, symnode in base.names.items():
+            node = symnode.node
+            if isinstance(node, OverloadedFuncDef):
+                # Unwrap an overloaded function definition. We can just
+                # check arbitrarily the first overload item. If the
+                # different items have a different abstract status, there
+                # should be an error reported elsewhere.
+                func = node.items[0]  # type: Optional[Node]
+            else:
+                func = node
+            if isinstance(func, Decorator):
+                fdef = func.func
+                if fdef.is_abstract and name not in concrete:
+                    typ.is_abstract = True
+                    abstract.append(name)
+                    if base is typ:
+                        abstract_in_this_class.append(name)
+            elif isinstance(node, Var):
+                if node.is_abstract_var and name not in concrete:
+                    typ.is_abstract = True
+                    abstract.append(name)
+                    if base is typ:
+                        abstract_in_this_class.append(name)
+            concrete.add(name)
+    # In stubs, abstract classes need to be explicitly marked because it is too
+    # easy to accidentally leave a concrete class abstract by forgetting to
+    # implement some methods.
+    typ.abstract_attributes = sorted(abstract)
+    if is_stub_file:
+        if typ.declared_metaclass and typ.declared_metaclass.type.fullname() == 'abc.ABCMeta':
+            return
+        if typ.is_protocol:
+            return
+        if abstract and not abstract_in_this_class:
+            def report(message: str, severity: str) -> None:
+                errors.report(typ.line, typ.column, message, severity=severity)
+
+            attrs = ", ".join('"{}"'.format(attr) for attr in sorted(abstract))
+            report("Class {} has abstract attributes {}".format(typ.fullname(), attrs), 'error')
+            report("If it is meant to be abstract, add 'abc.ABCMeta' as an explicit metaclass",
+                   'note')

mypy/newsemanal/semanal_infer.py

@@ -0,0 +1,114 @@
+"""Simple type inference for decorated functions during semantic analysis."""
+
+from typing import Optional
+
+from mypy.nodes import Expression, Decorator, CallExpr, FuncDef, RefExpr, Var, ARG_POS
+from mypy.types import Type, CallableType, AnyType, TypeOfAny, TypeVarType, function_type
+from mypy.typevars import has_no_typevars
+from mypy.newsemanal.semanal_shared import SemanticAnalyzerInterface
+
+
+def infer_decorator_signature_if_simple(dec: Decorator,
+                                        analyzer: SemanticAnalyzerInterface) -> None:
+    """Try to infer the type of the decorated function.
+
+    This lets us resolve additional references to decorated functions
+    during type checking. Otherwise the type might not be available
+    when we need it, since module top levels can't be deferred.
+
+    This basically uses a simple special-purpose type inference
+    engine just for decorators.
+    """
+    if dec.var.is_property:
+        # Decorators are expected to have a callable type (it's a little odd).
+        if dec.func.type is None:
+            dec.var.type = CallableType(
+                [AnyType(TypeOfAny.special_form)],
+                [ARG_POS],
+                [None],
+                AnyType(TypeOfAny.special_form),
+                analyzer.named_type('__builtins__.function'),
+                name=dec.var.name())
+        elif isinstance(dec.func.type, CallableType):
+            dec.var.type = dec.func.type
+        return
+    decorator_preserves_type = True
+    for expr in dec.decorators:
+        preserve_type = False
+        if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef):
+            if expr.node.type and is_identity_signature(expr.node.type):
+                preserve_type = True
+        if not preserve_type:
+            decorator_preserves_type = False
+            break
+    if decorator_preserves_type:
+        # No non-identity decorators left. We can trivially infer the type
+        # of the function here.
+        dec.var.type = function_type(dec.func, analyzer.named_type('__builtins__.function'))
+    if dec.decorators:
+        return_type = calculate_return_type(dec.decorators[0])
+        if return_type and isinstance(return_type, AnyType):
+            # The outermost decorator will return Any so we know the type of the
+            # decorated function.
+            dec.var.type = AnyType(TypeOfAny.from_another_any, source_any=return_type)
+        sig = find_fixed_callable_return(dec.decorators[0])
+        if sig:
+            # The outermost decorator always returns the same kind of function,
+            # so we know that this is the type of the decorated function.
+            orig_sig = function_type(dec.func, analyzer.named_type('__builtins__.function'))
+            sig.name = orig_sig.items()[0].name
+            dec.var.type = sig
+
+
+def is_identity_signature(sig: Type) -> bool:
+    """Is type a callable of form T -> T (where T is a type variable)?"""
+    if isinstance(sig, CallableType) and sig.arg_kinds == [ARG_POS]:
+        if isinstance(sig.arg_types[0], TypeVarType) and isinstance(sig.ret_type, TypeVarType):
+            return sig.arg_types[0].id == sig.ret_type.id
+    return False
+
+
+def calculate_return_type(expr: Expression) -> Optional[Type]:
+    """Return the return type if we can calculate it.
+
+    This only uses information available during semantic analysis so this
+    will sometimes return None because of insufficient information (as
+    type inference hasn't run yet).
+    """
+    if isinstance(expr, RefExpr):
+        if isinstance(expr.node, FuncDef):
+            typ = expr.node.type
+            if typ is None:
+                # No signature -> default to Any.
+                return AnyType(TypeOfAny.unannotated)
+            # Explicit Any return?
+            if isinstance(typ, CallableType):
+                return typ.ret_type
+            return None
+        elif isinstance(expr.node, Var):
+            return expr.node.type
+    elif isinstance(expr, CallExpr):
+        return calculate_return_type(expr.callee)
+    return None
+
+
+def find_fixed_callable_return(expr: Expression) -> Optional[CallableType]:
+    """Return the return type, if expression refers to a callable that returns a callable.
+
+    But only do this if the return type has no type variables. Return None otherwise.
+    This approximates things a lot as this is supposed to be called before type checking
+    when full type information is not available yet.
+    """
+    if isinstance(expr, RefExpr):
+        if isinstance(expr.node, FuncDef):
+            typ = expr.node.type
+            if typ:
+                if isinstance(typ, CallableType) and has_no_typevars(typ.ret_type):
+                    if isinstance(typ.ret_type, CallableType):
+                        return typ.ret_type
+    elif isinstance(expr, CallExpr):
+        t = find_fixed_callable_return(expr.callee)
+        if t:
+            if isinstance(t.ret_type, CallableType):
+                return t.ret_type
+    return None

mypy/newsemanal/semanal_main.py

@@ -31,11 +31,14 @@
 )
 from mypy.newsemanal.semanal_typeargs import TypeArgumentAnalyzer
 from mypy.state import strict_optional_set
+from mypy.newsemanal.semanal import NewSemanticAnalyzer
+from mypy.newsemanal.semanal_abstract import calculate_abstract_status
+from mypy.errors import Errors
+from mypy.newsemanal.semanal_infer import infer_decorator_signature_if_simple
 
 MYPY = False
 if MYPY:
     from mypy.build import Graph, State
-    from mypy.newsemanal.semanal import NewSemanticAnalyzer
 
 
 # Perform up to this many semantic analysis iterations until giving up trying to bind all names.
@@ -46,7 +49,7 @@
 core_modules = ['typing', 'builtins', 'abc', 'collections']
 
 
-def semantic_analysis_for_scc(graph: 'Graph', scc: List[str]) -> None:
+def semantic_analysis_for_scc(graph: 'Graph', scc: List[str], errors: Errors) -> None:
     """Perform semantic analysis for all modules in a SCC (import cycle).
 
     Assume that reachability analysis has already been performed.
@@ -56,7 +59,8 @@ def semantic_analysis_for_scc(graph: 'Graph', scc: List[str]) -> None:
     # before functions. This limitation is unlikely to go away soon.
     process_top_levels(graph, scc)
     process_functions(graph, scc)
-    check_type_arguments(graph, scc)
+    check_type_arguments(graph, scc, errors)
+    process_abstract_status(graph, scc, errors)
 
 
 def process_top_levels(graph: 'Graph', scc: List[str]) -> None:
@@ -183,22 +187,31 @@ def semantic_analyze_target(target: str,
                                    fnam=tree.path,
                                    options=state.options,
                                    active_type=active_type):
-            if isinstance(node, Decorator):
+            refresh_node = node
+            if isinstance(refresh_node, Decorator):
                 # Decorator expressions will be processed as part of the module top level.
-                node = node.func
-            analyzer.refresh_partial(node, [], final_iteration)
+                refresh_node = refresh_node.func
+            analyzer.refresh_partial(refresh_node, [], final_iteration)
+            if isinstance(node, Decorator):
+                infer_decorator_signature_if_simple(node, analyzer)
     if analyzer.deferred:
         return [target], analyzer.incomplete
     else:
         return [], analyzer.incomplete
 
 
-def check_type_arguments(graph: 'Graph', scc: List[str]) -> None:
+def check_type_arguments(graph: 'Graph', scc: List[str], errors: Errors) -> None:
     for module in scc:
         state = graph[module]
-        errors = state.manager.errors
         assert state.tree
         analyzer = TypeArgumentAnalyzer(errors)
         with state.wrap_context():
             with strict_optional_set(state.options.strict_optional):
                 state.tree.accept(analyzer)
+
+
+def process_abstract_status(graph: 'Graph', scc: List[str], errors: Errors) -> None:
+    for module in scc:
+        tree = graph[module].tree
+        assert tree
+        calculate_abstract_status(tree, errors)