Runtime implementation of TypedDict extension (#2552)

This was initially proposed in python/typing#322. It works on Python 2 and 3.

Author: ilevkivskyi

extensions/mypy_extensions.py

@@ -7,16 +7,84 @@
 
 # NOTE: This module must support Python 2.7 in addition to Python 3.x
 
+import sys
+# _type_check is NOT a part of public typing API, it is used here only to mimic
+# the (convenient) behavior of types provided by typing module.
+from typing import _type_check  # type: ignore
 
-def TypedDict(typename, fields):
-    """TypedDict creates a dictionary type that expects all of its
+
+def _check_fails(cls, other):
+    try:
+        if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']:
+            # Typed dicts are only for static structural subtyping.
+            raise TypeError('TypedDict does not support instance and class checks')
+    except (AttributeError, ValueError):
+        pass
+    return False
+
+def _dict_new(cls, *args, **kwargs):
+    return dict(*args, **kwargs)
+
+def _typeddict_new(cls, _typename, _fields=None, **kwargs):
+    if _fields is None:
+        _fields = kwargs
+    elif kwargs:
+        raise TypeError("TypedDict takes either a dict or keyword arguments,"
+                        " but not both")
+    return _TypedDictMeta(_typename, (), {'__annotations__': dict(_fields)})
+
+class _TypedDictMeta(type):
+    def __new__(cls, name, bases, ns):
+        # Create new typed dict class object.
+        # This method is called directly when TypedDict is subclassed,
+        # or via _typeddict_new when TypedDict is instantiated. This way
+        # TypedDict supports all three syntaxes described in its docstring.
+        # Subclasses and instanes of TypedDict return actual dictionaries
+        # via _dict_new.
+        ns['__new__'] = _typeddict_new if name == 'TypedDict' else _dict_new
+        tp_dict = super(_TypedDictMeta, cls).__new__(cls, name, (dict,), ns)
+        try:
+            # Setting correct module is necessary to make typed dict classes pickleable.
+            tp_dict.__module__ = sys._getframe(2).f_globals.get('__name__', '__main__')
+        except (AttributeError, ValueError):
+            pass
+        anns = ns.get('__annotations__', {})
+        msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type"
+        anns = {n: _type_check(tp, msg) for n, tp in anns.items()}
+        for base in bases:
+            anns.update(base.__dict__.get('__annotations__', {}))
+        tp_dict.__annotations__ = anns
+        return tp_dict
+
+    __instancecheck__ = __subclasscheck__ = _check_fails
+
+
+TypedDict = _TypedDictMeta('TypedDict', (dict,), {})
+TypedDict.__module__ = __name__
+TypedDict.__doc__ = \
+    """A simple typed name space. At runtime it is equivalent to a plain dict.
+
+    TypedDict creates a dictionary type that expects all of its
     instances to have a certain set of keys, with each key
     associated with a value of a consistent type. This expectation
     is not checked at runtime but is only enforced by typecheckers.
-    """
-    def new_dict(*args, **kwargs):
-        return dict(*args, **kwargs)
+    Usage::
+
+        Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str})
+        a: Point2D = {'x': 1, 'y': 2, 'label': 'good'}  # OK
+        b: Point2D = {'z': 3, 'label': 'bad'}           # Fails type check
+        assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first')
+
+    The type info could be accessed via Point2D.__annotations__. TypedDict
+    supports two additional equivalent forms::
 
-    new_dict.__name__ = typename
-    new_dict.__supertype__ = dict
-    return new_dict
+        Point2D = TypedDict('Point2D', x=int, y=int, label=str)
+
+        class Point2D(TypedDict):
+            x: int
+            y: int
+            label: str
+
+    The latter syntax is only supported in Python 3.6+, while two other
+    syntax forms work for Python 2.7 and 3.2+
+    """

mypy/semanal.py

@@ -1873,8 +1873,9 @@ def process_typeddict_definition(self, s: AssignmentStmt) -> None:
             return
         # Yes, it's a valid TypedDict definition. Add it to the symbol table.
         node = self.lookup(name, s)
-        node.kind = GDEF   # TODO locally defined TypedDict
-        node.node = typed_dict
+        if node:
+            node.kind = GDEF   # TODO locally defined TypedDict
+            node.node = typed_dict
 
     def check_typeddict(self, node: Expression, var_name: str = None) -> Optional[TypeInfo]:
         """Check if a call defines a TypedDict.

mypy/test/testextensions.py

@@ -0,0 +1,119 @@
+import sys
+import pickle
+import typing
+try:
+    import collections.abc as collections_abc
+except ImportError:
+    import collections as collections_abc  # type: ignore # PY32 and earlier
+from unittest import TestCase, main, skipUnless
+sys.path[0:0] = ['extensions']
+from mypy_extensions import TypedDict
+
+
+class BaseTestCase(TestCase):
+
+    def assertIsSubclass(self, cls, class_or_tuple, msg=None):
+        if not issubclass(cls, class_or_tuple):
+            message = '%r is not a subclass of %r' % (cls, class_or_tuple)
+            if msg is not None:
+                message += ' : %s' % msg
+            raise self.failureException(message)
+
+    def assertNotIsSubclass(self, cls, class_or_tuple, msg=None):
+        if issubclass(cls, class_or_tuple):
+            message = '%r is a subclass of %r' % (cls, class_or_tuple)
+            if msg is not None:
+                message += ' : %s' % msg
+            raise self.failureException(message)
+
+
+PY36 = sys.version_info[:2] >= (3, 6)
+
+PY36_TESTS = """
+Label = TypedDict('Label', [('label', str)])
+
+class Point2D(TypedDict):
+    x: int
+    y: int
+
+class LabelPoint2D(Point2D, Label): ...
+"""
+
+if PY36:
+    exec(PY36_TESTS)
+
+
+class TypedDictTests(BaseTestCase):
+
+    def test_basics_iterable_syntax(self):
+        Emp = TypedDict('Emp', {'name': str, 'id': int})
+        self.assertIsSubclass(Emp, dict)
+        self.assertIsSubclass(Emp, typing.MutableMapping)
+        self.assertNotIsSubclass(Emp, collections_abc.Sequence)
+        jim = Emp(name='Jim', id=1)
+        self.assertIs(type(jim), dict)
+        self.assertEqual(jim['name'], 'Jim')
+        self.assertEqual(jim['id'], 1)
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp.__module__, 'mypy.test.testextensions')
+        self.assertEqual(Emp.__bases__, (dict,))
+        self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
+
+    def test_basics_keywords_syntax(self):
+        Emp = TypedDict('Emp', name=str, id=int)
+        self.assertIsSubclass(Emp, dict)
+        self.assertIsSubclass(Emp, typing.MutableMapping)
+        self.assertNotIsSubclass(Emp, collections_abc.Sequence)
+        jim = Emp(name='Jim', id=1)  # type: ignore # mypy doesn't support keyword syntax yet
+        self.assertIs(type(jim), dict)
+        self.assertEqual(jim['name'], 'Jim')
+        self.assertEqual(jim['id'], 1)
+        self.assertEqual(Emp.__name__, 'Emp')
+        self.assertEqual(Emp.__module__, 'mypy.test.testextensions')
+        self.assertEqual(Emp.__bases__, (dict,))
+        self.assertEqual(Emp.__annotations__, {'name': str, 'id': int})
+
+    def test_typeddict_errors(self):
+        Emp = TypedDict('Emp', {'name': str, 'id': int})
+        self.assertEqual(TypedDict.__module__, 'mypy_extensions')
+        jim = Emp(name='Jim', id=1)
+        with self.assertRaises(TypeError):
+            isinstance({}, Emp)
+        with self.assertRaises(TypeError):
+            isinstance(jim, Emp)
+        with self.assertRaises(TypeError):
+            issubclass(dict, Emp)
+        with self.assertRaises(TypeError):
+            TypedDict('Hi', x=1)
+        with self.assertRaises(TypeError):
+            TypedDict('Hi', [('x', int), ('y', 1)])
+        with self.assertRaises(TypeError):
+            TypedDict('Hi', [('x', int)], y=int)
+
+    @skipUnless(PY36, 'Python 3.6 required')
+    def test_py36_class_syntax_usage(self):
+        self.assertEqual(LabelPoint2D.__annotations__, {'x': int, 'y': int, 'label': str})  # noqa
+        self.assertEqual(LabelPoint2D.__bases__, (dict,))  # noqa
+        self.assertNotIsSubclass(LabelPoint2D, typing.Sequence)  # noqa
+        not_origin = Point2D(x=0, y=1)  # noqa
+        self.assertEqual(not_origin['x'], 0)
+        self.assertEqual(not_origin['y'], 1)
+        other = LabelPoint2D(x=0, y=1, label='hi')  # noqa
+        self.assertEqual(other['label'], 'hi')
+
+    def test_pickle(self):
+        global EmpD  # pickle wants to reference the class by name
+        EmpD = TypedDict('EmpD', name=str, id=int)
+        jane = EmpD({'name': 'jane', 'id': 37})
+        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+            z = pickle.dumps(jane, proto)
+            jane2 = pickle.loads(z)
+            self.assertEqual(jane2, jane)
+            self.assertEqual(jane2, {'name': 'jane', 'id': 37})
+            ZZ = pickle.dumps(EmpD, proto)
+            EmpDnew = pickle.loads(ZZ)
+            self.assertEqual(EmpDnew({'name': 'jane', 'id': 37}), jane)
+
+
+if __name__ == '__main__':
+    main()

runtests.py

@@ -207,7 +207,7 @@ def add_imports(driver: Driver) -> None:
 
 
 PYTEST_FILES = ['mypy/test/{}.py'.format(name) for name in [
-    'testcheck',
+    'testcheck', 'testextensions',
 ]]