Prohibit list[int], etc (those fail at runtime) (#2869)

Fixes #2428

All of the following are now allowed, but fail at runtime:
```
list[int]
dict[int, str]
set[int]
tuple[int]
frozenset[int]
enumerate[int]
collections.defaultdict[int, str]
collections.Counter[str]
collections.ChainMap[str, str]
```
I prohibit those by simply tracking whether a corresponding symbol table node was normalized or not.

@gvanrossum I make an exclusion for stubs, because a have found dozens of places where this is used in typeshed, if you think that it also makes sense to prohibit this in stubs, then I will make an additional PR to typeshed.

Author: ilevkivskyi

mypy/build.py

@@ -21,7 +21,9 @@
 from os.path import dirname, basename
 
 from typing import (AbstractSet, Dict, Iterable, Iterator, List,
-                    NamedTuple, Optional, Set, Tuple, Union)
+                    NamedTuple, Optional, Set, Tuple, Union, TYPE_CHECKING)
+if TYPE_CHECKING:
+    from typing import Deque
 
 from mypy.nodes import (MypyFile, Node, ImportBase, Import, ImportFrom, ImportAll)
 from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass
@@ -1618,7 +1620,7 @@ def load_graph(sources: List[BuildSource], manager: BuildManager) -> Graph:
     # The deque is used to implement breadth-first traversal.
     # TODO: Consider whether to go depth-first instead.  This may
     # affect the order in which we process files within import cycles.
-    new = collections.deque()  # type: collections.deque[State]
+    new = collections.deque()  # type: Deque[State]
     entry_points = set()  # type: Set[str]
     # Seed the graph with the initial root sources.
     for bs in sources:

mypy/nodes.py

@@ -83,6 +83,7 @@ def get_column(self) -> int: pass
     'typing.List': '__builtins__.list',
     'typing.Dict': '__builtins__.dict',
     'typing.Set': '__builtins__.set',
+    'typing.FrozenSet': '__builtins__.frozenset',
 }
 
 reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias)
@@ -95,6 +96,15 @@ def get_column(self) -> int: pass
     'typing.Deque': '__mypy_collections__.deque',
 }
 
+reverse_collection_aliases = dict((name.replace('__mypy_collections__', 'collections'), alias)
+                                  for alias, name in
+                                  collections_type_aliases.items())  # type: Dict[str, str]
+
+nongen_builtins = {'builtins.tuple': 'typing.Tuple',
+                   'builtins.enumerate': ''}
+nongen_builtins.update(reverse_type_aliases)
+nongen_builtins.update(reverse_collection_aliases)
+
 
 # See [Note Literals and literal_hash] below
 Key = tuple
@@ -2160,17 +2170,20 @@ class SymbolTableNode:
     # For deserialized MODULE_REF nodes, the referenced module name;
     # for other nodes, optionally the name of the referenced object.
     cross_ref = None  # type: Optional[str]
+    # Was this node created by normalіze_type_alias?
+    normalized = False  # type: bool
 
     def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None,
                  typ: 'mypy.types.Type' = None,
                  tvar_def: 'mypy.types.TypeVarDef' = None,
-                 module_public: bool = True) -> None:
+                 module_public: bool = True, normalized: bool = False) -> None:
         self.kind = kind
         self.node = node
         self.type_override = typ
         self.mod_id = mod_id
         self.tvar_def = tvar_def
         self.module_public = module_public
+        self.normalized = normalized
 
     @property
     def fullname(self) -> str:

mypy/semanal.py

@@ -65,7 +65,7 @@
     SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr,
     YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, AwaitExpr,
     IntExpr, FloatExpr, UnicodeExpr, EllipsisExpr, TempNode,
-    COVARIANT, CONTRAVARIANT, INVARIANT, UNBOUND_IMPORTED, LITERAL_YES,
+    COVARIANT, CONTRAVARIANT, INVARIANT, UNBOUND_IMPORTED, LITERAL_YES, nongen_builtins,
     collections_type_aliases, get_member_expr_fullname,
 )
 from mypy.typevars import has_no_typevars, fill_typevars
@@ -79,7 +79,9 @@
     TupleType, UnionType, StarType, EllipsisType, function_type, TypedDictType,
 )
 from mypy.nodes import implicit_module_attrs
-from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias
+from mypy.typeanal import (
+    TypeAnalyser, TypeAnalyserPass3, analyze_type_alias, no_subscript_builtin_alias,
+)
 from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
 from mypy.sametypes import is_same_type
 from mypy.options import Options
@@ -1248,7 +1250,8 @@ def visit_import_from(self, imp: ImportFrom) -> None:
                 symbol = SymbolTableNode(node.kind, node.node,
                                          self.cur_mod_id,
                                          node.type_override,
-                                         module_public=module_public)
+                                         module_public=module_public,
+                                         normalized=node.normalized)
                 self.add_symbol(imported_id, symbol, imp)
             elif module and not missing:
                 # Missing attribute.
@@ -1284,13 +1287,19 @@ def process_import_over_existing_name(self,
 
     def normalize_type_alias(self, node: SymbolTableNode,
                              ctx: Context) -> SymbolTableNode:
+        normalized = False
         if node.fullname in type_aliases:
             # Node refers to an aliased type such as typing.List; normalize.
             node = self.lookup_qualified(type_aliases[node.fullname], ctx)
+            normalized = True
         if node.fullname in collections_type_aliases:
             # Similar, but for types from the collections module like typing.DefaultDict
             self.add_module_symbol('collections', '__mypy_collections__', False, ctx)
             node = self.lookup_qualified(collections_type_aliases[node.fullname], ctx)
+            normalized = True
+        if normalized:
+            node = SymbolTableNode(node.kind, node.node,
+                                   node.mod_id, node.type_override, normalized=True)
         return node
 
     def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str:
@@ -1326,7 +1335,8 @@ def visit_import_all(self, i: ImportAll) -> None:
                             continue
                     self.add_symbol(name, SymbolTableNode(node.kind, node.node,
                                                           self.cur_mod_id,
-                                                          node.type_override), i)
+                                                          node.type_override,
+                                                          normalized=node.normalized), i)
         else:
             # Don't add any dummy symbols for 'from x import *' if 'x' is unknown.
             pass
@@ -1365,7 +1375,8 @@ def anal_type(self, t: Type, allow_tuple_literal: bool = False,
                              self.lookup_fully_qualified,
                              self.fail,
                              aliasing=aliasing,
-                             allow_tuple_literal=allow_tuple_literal)
+                             allow_tuple_literal=allow_tuple_literal,
+                             allow_unnormalized=self.is_stub_file)
             return t.accept(a)
         else:
             return None
@@ -1388,7 +1399,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
                 res = analyze_type_alias(s.rvalue,
                                          self.lookup_qualified,
                                          self.lookup_fully_qualified,
-                                         self.fail)
+                                         self.fail, allow_unnormalized=True)
                 if res and (not isinstance(res, Instance) or res.args):
                     # TODO: What if this gets reassigned?
                     name = s.lvalues[0]
@@ -1465,7 +1476,9 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None:
                         # TODO: We should record the fact that this is a variable
                         #       that refers to a type, rather than making this
                         #       just an alias for the type.
-                        self.globals[lvalue.name].node = node
+                        sym = self.lookup_type_node(rvalue)
+                        if sym:
+                            self.globals[lvalue.name] = sym
 
     def analyze_lvalue(self, lval: Lvalue, nested: bool = False,
                        add_global: bool = False,
@@ -2705,7 +2718,7 @@ def visit_index_expr(self, expr: IndexExpr) -> None:
             res = analyze_type_alias(expr,
                                      self.lookup_qualified,
                                      self.lookup_fully_qualified,
-                                     self.fail)
+                                     self.fail, allow_unnormalized=self.is_stub_file)
             expr.analyzed = TypeAliasExpr(res, fallback=self.alias_fallback(res),
                                           in_runtime=True)
         elif refers_to_class_or_function(expr.base):
@@ -2726,9 +2739,23 @@ def visit_index_expr(self, expr: IndexExpr) -> None:
                 types.append(typearg)
             expr.analyzed = TypeApplication(expr.base, types)
             expr.analyzed.line = expr.line
+            # list, dict, set are not directly subscriptable
+            n = self.lookup_type_node(expr.base)
+            if n and not n.normalized and n.fullname in nongen_builtins:
+                self.fail(no_subscript_builtin_alias(n.fullname, propose_alt=False), expr)
         else:
             expr.index.accept(self)
 
+    def lookup_type_node(self, expr: Expression) -> Optional[SymbolTableNode]:
+        try:
+            t = expr_to_unanalyzed_type(expr)
+        except TypeTranslationError:
+            return None
+        if isinstance(t, UnboundType):
+            n = self.lookup_qualified(t.name, expr)
+            return n
+        return None
+
     def visit_slice_expr(self, expr: SliceExpr) -> None:
         if expr.begin_index:
             expr.begin_index.accept(self)

mypy/typeanal.py

@@ -12,7 +12,7 @@
 from mypy.nodes import (
     BOUND_TVAR, UNBOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED,
     TypeInfo, Context, SymbolTableNode, Var, Expression,
-    IndexExpr, RefExpr
+    IndexExpr, RefExpr, nongen_builtins,
 )
 from mypy.sametypes import is_same_type
 from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
@@ -33,7 +33,8 @@
 def analyze_type_alias(node: Expression,
                        lookup_func: Callable[[str, Context], SymbolTableNode],
                        lookup_fqn_func: Callable[[str], SymbolTableNode],
-                       fail_func: Callable[[str, Context], None]) -> Type:
+                       fail_func: Callable[[str, Context], None],
+                       allow_unnormalized: bool = False) -> Type:
     """Return type if node is valid as a type alias rvalue.
 
     Return None otherwise. 'node' must have been semantically analyzed.
@@ -68,10 +69,19 @@ def analyze_type_alias(node: Expression,
     except TypeTranslationError:
         fail_func('Invalid type alias', node)
         return None
-    analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func, aliasing=True)
+    analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func, aliasing=True,
+                            allow_unnormalized=allow_unnormalized)
     return type.accept(analyzer)
 
 
+def no_subscript_builtin_alias(name: str, propose_alt: bool = True) -> str:
+    msg = '"{}" is not subscriptable'.format(name.split('.')[-1])
+    replacement = nongen_builtins[name]
+    if replacement and propose_alt:
+        msg += ', use "{}" instead'.format(replacement)
+    return msg
+
+
 class TypeAnalyser(TypeVisitor[Type]):
     """Semantic analyzer for types (semantic analysis pass 2).
 
@@ -83,14 +93,16 @@ def __init__(self,
                  lookup_fqn_func: Callable[[str], SymbolTableNode],
                  fail_func: Callable[[str, Context], None], *,
                  aliasing: bool = False,
-                 allow_tuple_literal: bool = False) -> None:
+                 allow_tuple_literal: bool = False,
+                 allow_unnormalized: bool = False) -> None:
         self.lookup = lookup_func
         self.lookup_fqn_func = lookup_fqn_func
         self.fail = fail_func
         self.aliasing = aliasing
         self.allow_tuple_literal = allow_tuple_literal
         # Positive if we are analyzing arguments of another (outer) type
         self.nesting_level = 0
+        self.allow_unnormalized = allow_unnormalized
 
     def visit_unbound_type(self, t: UnboundType) -> Type:
         if t.optional:
@@ -106,6 +118,9 @@ def visit_unbound_type(self, t: UnboundType) -> Type:
                     self.fail('Internal error (node is None, kind={})'.format(sym.kind), t)
                 return AnyType()
             fullname = sym.node.fullname()
+            if (fullname in nongen_builtins and t.args and
+                    not sym.normalized and not self.allow_unnormalized):
+                self.fail(no_subscript_builtin_alias(fullname), t)
             if sym.kind == BOUND_TVAR:
                 if len(t.args) > 0:
                     self.fail('Type variable "{}" used with arguments'.format(

test-data/unit/check-generics.test

@@ -978,6 +978,30 @@ Bad = A[int] # type: ignore
 reveal_type(Bad) # E: Revealed type is 'Any'
 [out]
 
+[case testNoSubscriptionOfBuiltinAliases]
+from typing import List, TypeVar
+
+list[int]() # E: "list" is not subscriptable
+
+ListAlias = List
+def fun() -> ListAlias[int]:
+    pass
+
+reveal_type(fun())  # E: Revealed type is 'builtins.list[builtins.int]'
+
+BuiltinAlias = list
+BuiltinAlias[int]() # E: "list" is not subscriptable
+
+#check that error is reported only once, and type is still stored
+T = TypeVar('T')
+BadGenList = list[T] # E: "list" is not subscriptable
+
+reveal_type(BadGenList[int]()) # E: Revealed type is 'builtins.list[builtins.int*]'
+reveal_type(BadGenList()) # E: Revealed type is 'builtins.list[Any]'
+
+[builtins fixtures/list.pyi]
+[out]
+
 
 -- Simplified declaration of generics
 -- ----------------------------------

test-data/unit/pythoneval.test

@@ -1100,6 +1100,29 @@ _program.py:9: error: Incompatible types in assignment (expression has type "int
 _program.py:19: error: List item 0 has incompatible type "Tuple[str, List[None]]"
 _program.py:23: error: Invalid index type "str" for "dict"; expected type "int"
 
+[case testNoSubcriptionOfStdlibCollections]
+import collections
+from collections import Counter
+from typing import TypeVar
+
+collections.defaultdict[int, str]()
+Counter[int]()
+
+T = TypeVar('T')
+DDint = collections.defaultdict[T, int]
+
+d = DDint[str]()
+d[0] = 1
+
+def f(d: collections.defaultdict[int, str]) -> None:
+    ...
+[out]
+_program.py:5: error: "defaultdict" is not subscriptable
+_program.py:6: error: "Counter" is not subscriptable
+_program.py:9: error: "defaultdict" is not subscriptable
+_program.py:12: error: Invalid index type "int" for "dict"; expected type "str"
+_program.py:14: error: "defaultdict" is not subscriptable, use "typing.DefaultDict" instead
+
 [case testCollectionsAliases]
 import typing as t
 import collections as c