Merge branch 'namedtuple'

Author: JukkaL

lib-typing/3.2/test_typing.py

@@ -6,7 +6,8 @@
     AbstractGeneric, Protocol, Sized, Iterable, Iterator, Sequence,
     AbstractSet, Mapping, BinaryIO, TextIO, SupportsInt, SupportsFloat,
     SupportsAbs, SupportsRound, Reversible, Undefined, AnyStr, builtinclass,
-    cast, disjointclass, ducktype, forwardref, overload, typevar
+    cast, disjointclass, ducktype, forwardref, overload, typevar,
+    NamedTuple
 )
 
 
@@ -806,6 +807,11 @@ class A: pass
         self.assertIs(disjointclass(str)(A), A)
         self.assertIs(disjointclass('str')(A), A)
 
+    def test_NamedTuple(self):
+        n = NamedTuple('n', [('a', int), ('b', str)])
+        assert n(a=1, b='x') == (1, 'x')
+        assert n(1, 'x')[1] == 'x'
+
 
 @overload
 def global_overload(x:str) -> str:

lib-typing/3.2/typing.py

@@ -1,6 +1,7 @@
 """Static type checking helpers"""
 
 from abc import ABCMeta, abstractmethod, abstractproperty
+import collections
 import inspect
 import sys
 import re
@@ -19,6 +20,7 @@
     'IO',
     'List',
     'Match',
+    'NamedTuple',
     'Pattern',
     'Protocol',
     'Set',
@@ -173,10 +175,18 @@ def __getitem__(self, typeargs):
 Pattern = TypeAlias(type(re.compile('')))
 Match = TypeAlias(type(re.match('', '')))
 
+
 def union(x): return x
 
+
 Union = TypeAlias(union)
 
+
+def NamedTuple(typename, fields):
+    return collections.namedtuple(typename,
+                                  (name for name, type in fields))
+
+
 class typevar:
     def __init__(self, name, *, values=None):
         self.name = name

mypy/checker.py

@@ -17,7 +17,7 @@
     Context, ListComprehension, ConditionalExpr, GeneratorExpr,
     Decorator, SetExpr, PassStmt, TypeVarExpr, UndefinedExpr, PrintStmt,
     LITERAL_TYPE, BreakStmt, ContinueStmt, ComparisonExpr, StarExpr,
-    YieldFromExpr, YieldFromStmt
+    YieldFromExpr, YieldFromStmt, NamedTupleExpr
 )
 from mypy.nodes import function_type, method_type
 from mypy import nodes
@@ -1740,7 +1740,11 @@ def visit_type_application(self, e: TypeApplication) -> Type:
         return self.expr_checker.visit_type_application(e)
 
     def visit_type_var_expr(self, e: TypeVarExpr) -> Type:
-        # TODO Perhaps return a special type used for type variables only?
+        # TODO: Perhaps return a special type used for type variables only?
+        return AnyType()
+
+    def visit_namedtuple_expr(self, e: NamedTupleExpr) -> Type:
+        # TODO: Perhaps return a type object type?
         return AnyType()
 
     def visit_list_expr(self, e: ListExpr) -> Type:
@@ -1929,13 +1933,15 @@ def map_type_from_supertype(typ: Type, sub_info: TypeInfo,
 
     For example, assume
 
-      class D(Generic[S]) ...
-      class C(D[E[T]], Generic[T]) ...
+    . class D(Generic[S]) ...
+    . class C(D[E[T]], Generic[T]) ...
 
     Now S in the context of D would be mapped to E[T] in the context of C.
     """
     # Create the type of self in subtype, of form t[a1, ...].
     inst_type = self_type(sub_info)
+    if isinstance(inst_type, TupleType):
+        inst_type = inst_type.fallback
     # Map the type of self to supertype. This gets us a description of the
     # supertype type variables in terms of subtype variables, i.e. t[t1, ...]
     # so that any type variables in tN are to be interpreted in subtype

mypy/checkmember.py

@@ -239,7 +239,7 @@ def type_object_type(info: TypeInfo, builtin_type: Function[[str], Instance]) ->
 
       def [T, S](...) -> G[T, S],
 
-    where ... are argument types for the __init__ method.
+    where ... are argument types for the __init__ method (without the self argument).
     """
     init_method = info.get_method('__init__')
     if not init_method:

mypy/messages.py

@@ -195,6 +195,10 @@ def format_simple(self, typ: Type) -> str:
             # This is similar to non-generic instance types.
             return '"{}"'.format((cast(TypeVar, typ)).name)
         elif isinstance(typ, TupleType):
+            fallback = self.format_simple(typ.fallback)
+            if fallback != '"tuple"':
+                # Prefer the name of the fallback class (if not tuple), as it's more informative.
+                return fallback
             items = []
             for t in (cast(TupleType, typ)).items:
                 items.append(strip_quotes(self.format(t)))

mypy/nodes.py

@@ -1309,6 +1309,20 @@ def accept(self, visitor: NodeVisitor[T]) -> T:
         return visitor.visit_type_var_expr(self)
 
 
+class NamedTupleExpr(Node):
+    """Named tuple expression namedtuple(...)."""
+
+    # The class representation of this named tuple (its tuple_type attribute contains
+    # the tuple item types)
+    info = Undefined('TypeInfo')
+
+    def __init__(self, info: 'TypeInfo') -> None:
+        self.info = info
+
+    def accept(self, visitor: NodeVisitor[T]) -> T:
+        return visitor.visit_namedtuple_expr(self)
+
+
 class DucktypeExpr(Node):
     """Ducktype class decorator expression ducktype(...)."""
 
@@ -1383,6 +1397,13 @@ class TypeInfo(SymbolNode):
     # Duck type compatibility (ducktype decorator)
     ducktype = None  # type: mypy.types.Type
 
+    # Representation of a Tuple[...] base class, if the class has any
+    # (e.g., for named tuples). If this is not None, the actual Type
+    # object used for this class is not an Instance but a TupleType;
+    # the corresponding Instance is set as the fallback type of the
+    # tuple type.
+    tuple_type = None # type: mypy.types.TupleType
+
     def __init__(self, names: 'SymbolTable', defn: ClassDef) -> None:
         """Initialize a TypeInfo."""
         self.names = names

mypy/semanal.py

@@ -57,7 +57,7 @@
     FuncExpr, MDEF, FuncBase, Decorator, SetExpr, UndefinedExpr, TypeVarExpr,
     StrExpr, PrintStmt, ConditionalExpr, DucktypeExpr, DisjointclassExpr,
     ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases,
-    YieldFromStmt, YieldFromExpr
+    YieldFromStmt, YieldFromExpr, NamedTupleExpr
 )
 from mypy.visitor import NodeVisitor
 from mypy.traverser import TraverserVisitor
@@ -500,6 +500,11 @@ def analyze_base_classes(self, defn: ClassDef) -> None:
         bases = List[Instance]()
         for i in range(len(defn.base_types)):
             base = self.anal_type(defn.base_types[i])
+            if isinstance(base, TupleType):
+                if defn.info.tuple_type:
+                    self.fail("Class has two incompatible bases derived from tuple", defn)
+                defn.info.tuple_type = base
+                base = base.fallback
             if isinstance(base, Instance):
                 defn.base_types[i] = base
                 bases.append(base)
@@ -708,6 +713,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
                 self.store_declared_types(lvalue, s.type)
         self.check_and_set_up_type_alias(s)
         self.process_typevar_declaration(s)
+        self.process_namedtuple_definition(s)
 
     def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None:
         """Check if assignment creates a type alias and set it up as needed."""
@@ -952,14 +958,156 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> None:
                 return
         else:
             values = []
-        # Yes, it's a valid type variable definition!
+        # Yes, it's a valid type variable definition! Add it to the symbol table.
         node = self.lookup(name, s)
         node.kind = UNBOUND_TVAR
         typevar = TypeVarExpr(name, node.fullname, values)
         typevar.line = call.line
         call.analyzed = typevar
         node.node = typevar
 
+    def process_namedtuple_definition(self, s: AssignmentStmt) -> None:
+        """Check if s defines a namedtuple; if yes, store the definition in symbol table."""
+        if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr):
+            return
+        if not isinstance(s.rvalue, CallExpr):
+            return
+        call = cast(CallExpr, s.rvalue)
+        named_tuple = self.check_namedtuple(call)
+        if named_tuple is None:
+            return
+        # Yes, it's a valid namedtuple definition. Add it to the symbol table.
+        lvalue = cast(NameExpr, s.lvalues[0])
+        name = lvalue.name
+        node = self.lookup(name, s)
+        node.kind = GDEF   # TODO locally defined namedtuple
+        # TODO call.analyzed
+        node.node = named_tuple
+
+    def check_namedtuple(self, call: CallExpr) -> TypeInfo:
+        """Check if a call defines a namedtuple.
+
+        If it does, return the corresponding TypeInfo. Return None otherwise.
+
+        If the definition is invalid but looks like a namedtuple,
+        report errors but return (some) TypeInfo.
+        """
+        if not isinstance(call.callee, RefExpr):
+            return None
+        callee = cast(RefExpr, call.callee)
+        fullname = callee.fullname
+        if fullname not in ('collections.namedtuple', 'typing.NamedTuple'):
+            return None
+        items, types = self.parse_namedtuple_args(call, fullname)
+        if not items:
+            # Error. Construct dummy return value.
+            error_classdef = ClassDef('namedtuple', Block([]))
+            info = TypeInfo(SymbolTable(), error_classdef)
+        else:
+            listexpr = cast(ListExpr, call.args[1])
+            name = cast(StrExpr, call.args[0]).value
+            info = self.build_namedtuple_typeinfo(name, items, types)
+        call.analyzed = NamedTupleExpr(info).set_line(call.line)
+        return info
+
+    def parse_namedtuple_args(self, call: CallExpr,
+                              fullname: str) -> Tuple[List[str], List[Type]]:
+        # TODO Share code with check_argument_count in checkexpr.py?
+        args = call.args
+        if len(args) < 2:
+            return self.fail_namedtuple_arg("Too few arguments for namedtuple()", call)
+        if len(args) > 2:
+            return self.fail_namedtuple_arg("Too many arguments for namedtuple()", call)
+        if call.arg_kinds != [ARG_POS, ARG_POS]:
+            return self.fail_namedtuple_arg("Unexpected arguments to namedtuple()", call)
+        if not isinstance(args[0], StrExpr):
+            return self.fail_namedtuple_arg(
+                "namedtuple() expects a string literal as the first argument", call)
+        if not isinstance(args[1], ListExpr):
+            return self.fail_namedtuple_arg(
+                "List literal expected as the second argument to namedtuple()", call)
+        listexpr = cast(ListExpr, args[1])
+        if fullname == 'collections.namedtuple':
+            # The fields argument contains just names, with implicit Any types.
+            if any(not isinstance(item, StrExpr) for item in listexpr.items):
+                return self.fail_namedtuple_arg("String literal expected as namedtuple() item",
+                                                call)
+            items = [cast(StrExpr, item).value for item in listexpr.items]
+            types = [AnyType() for _ in listexpr.items]  # type: List[Type]
+        else:
+            # The fields argument contains (name, type) tuples.
+            items, types = self.parse_namedtuple_fields_with_types(listexpr.items, call)
+        return items, types
+
+    def parse_namedtuple_fields_with_types(self, nodes: List[Node],
+                                           context: Context) -> Tuple[List[str], List[Type]]:
+        items = []  # type: List[str]
+        types = []  # type: List[Type]
+        for item in nodes:
+            while isinstance(item, ParenExpr):
+                item = item.expr
+            if isinstance(item, TupleExpr):
+                if len(item.items) != 2:
+                    return self.fail_namedtuple_arg("Invalid NamedTuple field definition",
+                                                    item)
+                name, type_node = item.items
+                if isinstance(name, StrExpr):
+                    items.append(name.value)
+                else:
+                    return self.fail_namedtuple_arg("Invalid NamedTuple() field name", item)
+                try:
+                    type = expr_to_unanalyzed_type(type_node)
+                except TypeTranslationError:
+                    return self.fail_namedtuple_arg('Invalid field type', type_node)
+                types.append(self.anal_type(type))
+            else:
+                return self.fail_namedtuple_arg("Tuple expected as NamedTuple() field", item)
+        return items, types
+
+    def fail_namedtuple_arg(self, message: str, context: Context) -> Tuple[List[str], List[Type]]:
+        self.fail(message, context)
+        return [], []
+
+    def build_namedtuple_typeinfo(self, name: str, items: List[str],
+                                  types: List[Type]) -> TypeInfo:
+        symbols = SymbolTable()
+        class_def = ClassDef(name, Block([]))
+        class_def.fullname = self.qualified_name(name)
+        info = TypeInfo(symbols, class_def)
+        # Add named tuple items as attributes.
+        # TODO: Make them read-only.
+        for item, typ in zip(items, types):
+            var = Var(item)
+            var.info = info
+            var.type = typ
+            symbols[item] = SymbolTableNode(MDEF, var)
+        # Add a __init__ method.
+        init = self.make_namedtuple_init(info, items, types)
+        symbols['__init__'] = SymbolTableNode(MDEF, init)
+        info.tuple_type = TupleType(types, self.named_type('__builtins__.tuple'))
+        info.mro = [info] + info.tuple_type.fallback.type.mro
+        return info
+
+    def make_namedtuple_init(self, info: TypeInfo, items: List[str],
+                             types: List[Type]) -> FuncDef:
+        args = [Var(item) for item in items]
+        for arg, type in zip(args, types):
+            arg.type = type
+        # TODO: Make sure that the self argument name is not visible?
+        args = [Var('__self')] + args
+        arg_kinds = [ARG_POS] * (len(items) + 1)
+        signature = Callable([cast(Type, None)] + types,
+                             arg_kinds,
+                             ['__self'] + items,
+                             NoneTyp(),
+                             self.named_type('__builtins__.function'),
+                             name=info.name())
+        return FuncDef('__init__',
+                       args, arg_kinds,
+                       [None] * (len(items) + 1),
+                       Block([]),
+                       typ=signature)
+
     def analyze_types(self, items: List[Node]) -> List[Type]:
         result = List[Type]()
         for node in items:
@@ -1697,7 +1845,7 @@ def fail(self, msg: str, ctx: Context) -> None:
         self.errors.report(ctx.get_line(), msg)
 
 
-def self_type(typ: TypeInfo) -> Instance:
+def self_type(typ: TypeInfo) -> Union[Instance, TupleType]:
     """For a non-generic type, return instance type representing the type.
     For a generic G type with parameters T1, .., Tn, return G[T1, ..., Tn].
     """
@@ -1706,7 +1854,11 @@ def self_type(typ: TypeInfo) -> Instance:
         tv.append(TypeVar(typ.type_vars[i], i + 1,
                           typ.defn.type_vars[i].values,
                           typ.defn.type_vars[i].upper_bound))
-    return Instance(typ, tv)
+    inst = Instance(typ, tv)
+    if typ.tuple_type is None:
+        return inst
+    else:
+        return TupleType(typ.tuple_type.items, inst)
 
 
 @overload

mypy/strconv.py

@@ -394,6 +394,11 @@ def visit_type_var_expr(self, o):
         else:
             return 'TypeVarExpr:{}()'.format(o.line)
 
+    def visit_namedtuple_expr(self, o):
+        return 'NamedTupleExpr:{}({}, {})'.format(o.line,
+                                                  o.info.name(),
+                                                  o.info.tuple_type)
+
     def visit_ducktype_expr(self, o):
         return 'DucktypeExpr:{}({})'.format(o.line, o.type)
 

mypy/subtypes.py

@@ -132,6 +132,8 @@ def visit_tuple_type(self, left: TupleType) -> bool:
             for i in range(len(left.items)):
                 if not is_subtype(left.items[i], right.items[i]):
                     return False
+            if not is_subtype(left.fallback, right.fallback):
+                return False
             return True
         else:
             return False