[mypyc] Add a mypyc_attr to support interpreted children

This operates by generating a "shadow vtable" containing pointers to
glue methods that dispatch to the appropriate method via the C API. We
then install those shadow vtables in interpreted subclasses so that
overridden methods will be called.

This does not support directly inheriting from traits, which I think
will require generating vtables dynamically (and maybe some more
nonsense too.)

Closes #296. (Though I will file a follow-up for traits.)

Author: msullivan

mypy-requirements.txt

@@ -1,3 +1,3 @@
 typing_extensions>=3.7.4
-mypy_extensions>=0.4.0,<0.5.0
+mypy_extensions>=0.4.3,<0.5.0
 typed_ast>=1.4.0,<1.5.0

mypyc/emitclass.py

@@ -202,7 +202,7 @@ def emit_line() -> None:
         fields['tp_basicsize'] = base_size
 
     if generate_full:
-        emitter.emit_line('static PyObject *{}(void);'.format(setup_name))
+        emitter.emit_line('static PyObject *{}(PyTypeObject *type);'.format(setup_name))
         assert cl.ctor is not None
         emitter.emit_line(native_function_header(cl.ctor, emitter) + ';')
 
@@ -216,7 +216,15 @@ def emit_line() -> None:
         generate_dealloc_for_class(cl, dealloc_name, clear_name, emitter)
         emit_line()
         generate_native_getters_and_setters(cl, emitter)
-        vtable_name = generate_vtables(cl, vtable_setup_name, vtable_name, emitter)
+
+        if cl.allow_interpreted_children:
+            shadow_vtable_name = generate_vtables(
+                cl, vtable_setup_name + "_shadow", vtable_name + "_shadow", emitter, shadow=True
+            )  # type: Optional[str]
+            emit_line()
+        else:
+            shadow_vtable_name = None
+        vtable_name = generate_vtables(cl, vtable_setup_name, vtable_name, emitter, shadow=False)
         emit_line()
     if needs_getseters:
         generate_getseter_declarations(cl, emitter)
@@ -241,7 +249,8 @@ def emit_line() -> None:
 
     emitter.emit_line()
     if generate_full:
-        generate_setup_for_class(cl, setup_name, defaults_fn, vtable_name, emitter)
+        generate_setup_for_class(
+            cl, setup_name, defaults_fn, vtable_name, shadow_vtable_name, emitter)
         emitter.emit_line()
         generate_constructor_for_class(
             cl, cl.ctor, init_fn, setup_name, vtable_name, emitter)
@@ -344,7 +353,8 @@ def generate_native_getters_and_setters(cl: ClassIR,
 def generate_vtables(base: ClassIR,
                      vtable_setup_name: str,
                      vtable_name: str,
-                     emitter: Emitter) -> str:
+                     emitter: Emitter,
+                     shadow: bool) -> str:
     """Emit the vtables and vtable setup functions for a class.
 
     This includes both the primary vtable and any trait implementation vtables.
@@ -359,8 +369,9 @@ def generate_vtables(base: ClassIR,
     """
 
     def trait_vtable_name(trait: ClassIR) -> str:
-        return '{}_{}_trait_vtable'.format(
-            base.name_prefix(emitter.names), trait.name_prefix(emitter.names))
+        return '{}_{}_trait_vtable{}'.format(
+            base.name_prefix(emitter.names), trait.name_prefix(emitter.names),
+            '_shadow' if shadow else '')
 
     # Emit array definitions with enough space for all the entries
     emitter.emit_line('static CPyVTableItem {}[{}];'.format(
@@ -376,13 +387,16 @@ def trait_vtable_name(trait: ClassIR) -> str:
     emitter.emit_line('{}{}(void)'.format(NATIVE_PREFIX, vtable_setup_name))
     emitter.emit_line('{')
 
+    if base.allow_interpreted_children and not shadow:
+        emitter.emit_line('{}{}_shadow();'.format(NATIVE_PREFIX, vtable_setup_name))
+
     subtables = []
     for trait, vtable in base.trait_vtables.items():
         name = trait_vtable_name(trait)
-        generate_vtable(vtable, name, emitter, [])
+        generate_vtable(vtable, name, emitter, [], shadow)
         subtables.append((trait, name))
 
-    generate_vtable(base.vtable_entries, vtable_name, emitter, subtables)
+    generate_vtable(base.vtable_entries, vtable_name, emitter, subtables, shadow)
 
     emitter.emit_line('return 1;')
     emitter.emit_line('}')
@@ -393,7 +407,8 @@ def trait_vtable_name(trait: ClassIR) -> str:
 def generate_vtable(entries: VTableEntries,
                     vtable_name: str,
                     emitter: Emitter,
-                    subtables: List[Tuple[ClassIR, str]]) -> None:
+                    subtables: List[Tuple[ClassIR, str]],
+                    shadow: bool) -> None:
     emitter.emit_line('CPyVTableItem {}_scratch[] = {{'.format(vtable_name))
     if subtables:
         emitter.emit_line('/* Array of trait vtables */')
@@ -404,10 +419,11 @@ def generate_vtable(entries: VTableEntries,
 
     for entry in entries:
         if isinstance(entry, VTableMethod):
+            method = entry.shadow_method if shadow and entry.shadow_method else entry.method
             emitter.emit_line('(CPyVTableItem){}{}{},'.format(
                 emitter.get_group_prefix(entry.method.decl),
                 NATIVE_PREFIX,
-                entry.method.cname(emitter.names)))
+                method.cname(emitter.names)))
         else:
             cl, attr, is_setter = entry
             namer = native_setter_name if is_setter else native_getter_name
@@ -425,18 +441,27 @@ def generate_setup_for_class(cl: ClassIR,
                              func_name: str,
                              defaults_fn: Optional[FuncIR],
                              vtable_name: str,
+                             shadow_vtable_name: Optional[str],
                              emitter: Emitter) -> None:
     """Generate a native function that allocates an instance of a class."""
     emitter.emit_line('static PyObject *')
-    emitter.emit_line('{}(void)'.format(func_name))
+    emitter.emit_line('{}(PyTypeObject *type)'.format(func_name))
     emitter.emit_line('{')
     emitter.emit_line('{} *self;'.format(cl.struct_name(emitter.names)))
-    emitter.emit_line('self = ({struct} *){type_struct}->tp_alloc({type_struct}, 0);'.format(
-        struct=cl.struct_name(emitter.names),
-        type_struct=emitter.type_struct_name(cl)))
+    emitter.emit_line('self = ({struct} *)type->tp_alloc(type, 0);'.format(
+        struct=cl.struct_name(emitter.names)))
     emitter.emit_line('if (self == NULL)')
     emitter.emit_line('    return NULL;')
-    emitter.emit_line('self->vtable = {};'.format(vtable_name))
+
+    if shadow_vtable_name:
+        emitter.emit_line('if (type != {}) {{'.format(emitter.type_struct_name(cl)))
+        emitter.emit_line('self->vtable = {};'.format(shadow_vtable_name))
+        emitter.emit_line('} else {')
+        emitter.emit_line('self->vtable = {};'.format(vtable_name))
+        emitter.emit_line('}')
+    else:
+        emitter.emit_line('self->vtable = {};'.format(vtable_name))
+
     for base in reversed(cl.base_mro):
         for attr, rtype in base.attributes.items():
             emitter.emit_line('self->{} = {};'.format(
@@ -464,7 +489,7 @@ def generate_constructor_for_class(cl: ClassIR,
     """Generate a native function that allocates and initializes an instance of a class."""
     emitter.emit_line('{}'.format(native_function_header(fn, emitter)))
     emitter.emit_line('{')
-    emitter.emit_line('PyObject *self = {}();'.format(setup_name))
+    emitter.emit_line('PyObject *self = {}({});'.format(setup_name, emitter.type_struct_name(cl)))
     emitter.emit_line('if (self == NULL)')
     emitter.emit_line('    return NULL;')
     args = ', '.join(['self'] + [REG_PREFIX + arg.name for arg in fn.sig.args])
@@ -525,13 +550,15 @@ def generate_new_for_class(cl: ClassIR,
         '{}(PyTypeObject *type, PyObject *args, PyObject *kwds)'.format(func_name))
     emitter.emit_line('{')
     # TODO: Check and unbox arguments
-    emitter.emit_line('if (type != {}) {{'.format(emitter.type_struct_name(cl)))
-    emitter.emit_line(
-        'PyErr_SetString(PyExc_TypeError, "interpreted classes cannot inherit from compiled");')
-    emitter.emit_line('return NULL;')
-    emitter.emit_line('}')
+    if not cl.allow_interpreted_children:
+        emitter.emit_line('if (type != {}) {{'.format(emitter.type_struct_name(cl)))
+        emitter.emit_line(
+            'PyErr_SetString(PyExc_TypeError, "interpreted classes cannot inherit from compiled");'
+        )
+        emitter.emit_line('return NULL;')
+        emitter.emit_line('}')
 
-    emitter.emit_line('return {}();'.format(setup_name))
+    emitter.emit_line('return {}(type);'.format(setup_name))
     emitter.emit_line('}')
 
 

mypyc/genops.py

@@ -239,10 +239,55 @@ def is_dataclass(cdef: ClassDef) -> bool:
     return any(is_dataclass_decorator(d) for d in cdef.decorators)
 
 
-def is_extension_class(cdef: ClassDef) -> bool:
+def get_mypyc_attr_literal(e: Expression) -> Any:
+    """Convert an expression from a mypyc_attr decorator to a value.
+
+    Supports a pretty limited range."""
+    if isinstance(e, (StrExpr, IntExpr, FloatExpr)):
+        return e.value
+    elif isinstance(e, RefExpr) and e.fullname == 'builtins.True':
+        return True
+    elif isinstance(e, RefExpr) and e.fullname == 'builtins.False':
+        return False
+    elif isinstance(e, RefExpr) and e.fullname == 'builtins.None':
+        return None
+    return NotImplemented
+
+
+def get_mypyc_attr_call(d: Expression) -> Optional[CallExpr]:
+    """Check if an expression is a call to mypyc_attr and return it if so."""
+    if (
+        isinstance(d, CallExpr)
+        and isinstance(d.callee, RefExpr)
+        and d.callee.fullname == 'mypy_extensions.mypyc_attr'
+    ):
+        return d
+    return None
+
+
+def get_mypyc_attrs(stmt: Union[ClassDef, Decorator]) -> Dict[str, Any]:
+    """Collect all the mypyc_attr attributes on a class definition or a function."""
+    attrs = {}  # type: Dict[str, Any]
+    for dec in stmt.decorators:
+        d = get_mypyc_attr_call(dec)
+        if d:
+            for name, arg in zip(d.arg_names, d.args):
+                if name is None:
+                    if isinstance(arg, StrExpr):
+                        attrs[arg.value] = True
+                else:
+                    attrs[name] = get_mypyc_attr_literal(arg)
+
+    return attrs
+
 
-    if any(not is_trait_decorator(d) and not is_dataclass_decorator(d)
-           for d in cdef.decorators):
+def is_extension_class(cdef: ClassDef) -> bool:
+    if any(
+        not is_trait_decorator(d)
+        and not is_dataclass_decorator(d)
+        and not get_mypyc_attr_call(d)
+        for d in cdef.decorators
+    ):
         return False
     elif (cdef.info.metaclass_type and cdef.info.metaclass_type.type.fullname not in (
             'abc.ABCMeta', 'typing.TypingMeta', 'typing.GenericMeta')):
@@ -285,10 +330,11 @@ def specialize_parent_vtable(cls: ClassIR, parent: ClassIR) -> VTableEntries:
                 # TODO: emit a wrapper for __init__ that raises or something
                 if (is_same_method_signature(orig_parent_method.sig, child_method.sig)
                         or orig_parent_method.name == '__init__'):
-                    entry = VTableMethod(entry.cls, entry.name, child_method)
+                    entry = VTableMethod(entry.cls, entry.name, child_method, entry.shadow_method)
                 else:
                     entry = VTableMethod(entry.cls, entry.name,
-                                         defining_cls.glue_methods[(entry.cls, entry.name)])
+                                         defining_cls.glue_methods[(entry.cls, entry.name)],
+                                         entry.shadow_method)
         else:
             # If it is an attribute from a trait, we need to find out
             # the real class it got mixed in at and point to that.
@@ -346,7 +392,8 @@ def compute_vtable(cls: ClassIR) -> None:
             # TODO: don't generate a new entry when we overload without changing the type
             if fn == cls.get_method(fn.name):
                 cls.vtable[fn.name] = len(entries)
-                entries.append(VTableMethod(t, fn.name, fn))
+                shadow = cls.glue_methods.get((cls, fn.name))
+                entries.append(VTableMethod(t, fn.name, fn, shadow))
 
     # Compute vtables for all of the traits that the class implements
     if not cls.is_trait:
@@ -546,6 +593,10 @@ def prepare_class_def(path: str, module_name: str, cdef: ClassDef,
     ir = mapper.type_to_ir[cdef.info]
     info = cdef.info
 
+    attrs = get_mypyc_attrs(cdef)
+    if attrs.get("allow_interpreted_children") is True:
+        ir.allow_interpreted_children = True
+
     # We sort the table for determinism here on Python 3.5
     for name, node in sorted(info.names.items()):
         # Currenly all plugin generated methods are dummies and not included.
@@ -1169,20 +1220,26 @@ def handle_ext_method(self, cdef: ClassDef, fdef: FuncDef) -> None:
                     and not is_same_method_signature(class_ir.method_decls[name].sig,
                                                      cls.method_decls[name].sig)):
 
+                if cls is class_ir and not cls.allow_interpreted_children:
+                    continue
+
                 # TODO: Support contravariant subtyping in the input argument for
                 # property setters. Need to make a special glue method for handling this,
                 # similar to gen_glue_property.
 
-                if fdef.is_property:
-                    f = self.gen_glue_property(cls.method_decls[name].sig, func_ir, class_ir,
-                                               cls, fdef.line)
-                else:
-                    f = self.gen_glue_method(cls.method_decls[name].sig, func_ir, class_ir,
-                                             cls, fdef.line)
-
+                f = self.gen_glue(cls.method_decls[name].sig, func_ir, class_ir, cls, fdef)
                 class_ir.glue_methods[(cls, name)] = f
                 self.functions.append(f)
 
+        # If the class allows interpreted children, create glue
+        # methods that dispatch via the Python API. These will go in a
+        # "shadow vtable" that will be assigned to interpreted
+        # children.
+        if class_ir.allow_interpreted_children:
+            f = self.gen_glue(func_ir.sig, func_ir, class_ir, class_ir, fdef, do_py_ops=True)
+            class_ir.glue_methods[(class_ir, name)] = f
+            self.functions.append(f)
+
     def handle_non_ext_method(
             self, non_ext: NonExtClassInfo, cdef: ClassDef, fdef: FuncDef) -> None:
         # Perform the function of visit_method for methods inside non-extension classes.
@@ -1481,6 +1538,11 @@ def visit_class_def(self, cdef: ClassDef) -> None:
         if any(ir.base_mro[i].base != ir. base_mro[i + 1] for i in range(len(ir.base_mro) - 1)):
             self.error("Non-trait MRO must be linear", cdef.line)
 
+        if ir.allow_interpreted_children and any(
+            not parent.allow_interpreted_children for parent in ir.mro
+        ):
+            self.error("Parents must allow interpreted children also", cdef.line)
+
         # Currently, we only create non-extension classes for classes that are
         # decorated or inherit from Enum. Classes decorated with @trait do not
         # apply here, and are handled in a different way.
@@ -1707,8 +1769,24 @@ def visit_import_all(self, node: ImportAll) -> None:
             return
         self.gen_import(node.id, node.line)
 
+    def gen_glue(self, sig: FuncSignature, target: FuncIR,
+                 cls: ClassIR, base: ClassIR, fdef: FuncItem,
+                 *,
+                 do_py_ops: bool = False
+                 ) -> FuncIR:
+        if fdef.is_property:
+            return self.gen_glue_property(
+                cls.method_decls[fdef.name].sig, target, cls, base, fdef.line, do_py_ops
+            )
+        else:
+            return self.gen_glue_method(
+                cls.method_decls[fdef.name].sig, target, cls, base, fdef.line, do_py_ops
+            )
+
     def gen_glue_method(self, sig: FuncSignature, target: FuncIR,
-                        cls: ClassIR, base: ClassIR, line: int) -> FuncIR:
+                        cls: ClassIR, base: ClassIR, line: int,
+                        do_pycall: bool,
+                        ) -> FuncIR:
         """Generate glue methods that mediate between different method types in subclasses.
 
         For example, if we have:
@@ -1745,7 +1823,11 @@ def f(self, x: object) -> int: ...
         arg_names = [arg.name for arg in rt_args]
         arg_kinds = [concrete_arg_kind(arg.kind) for arg in rt_args]
 
-        retval = self.call(target.decl, args, arg_kinds, arg_names, line)
+        if do_pycall:
+            retval = self.py_method_call(
+                args[0], target.name, args[1:], line, arg_kinds[1:], arg_names[1:])
+        else:
+            retval = self.call(target.decl, args, arg_kinds, arg_names, line)
         retval = self.coerce(retval, sig.ret_type, line)
         self.add(Return(retval))
 
@@ -1758,7 +1840,8 @@ def f(self, x: object) -> int: ...
             blocks, env)
 
     def gen_glue_property(self, sig: FuncSignature, target: FuncIR, cls: ClassIR, base: ClassIR,
-                          line: int) -> FuncIR:
+                          line: int,
+                          do_pygetattr: bool) -> FuncIR:
         """Similarly to methods, properties of derived types can be covariantly subtyped. Thus,
         properties also require glue. However, this only requires the return type to change.
         Further, instead of a method call, an attribute get is performed."""
@@ -1767,7 +1850,10 @@ def gen_glue_property(self, sig: FuncSignature, target: FuncIR, cls: ClassIR, ba
         rt_arg = RuntimeArg(SELF_NAME, RInstance(cls))
         arg = self.read(self.add_self_to_env(cls), line)
         self.ret_types[-1] = sig.ret_type
-        retval = self.add(GetAttr(arg, target.name, line))
+        if do_pygetattr:
+            retval = self.py_get_attr(arg, target.name, line)
+        else:
+            retval = self.add(GetAttr(arg, target.name, line))
         retbox = self.coerce(retval, sig.ret_type, line)
         self.add(Return(retbox))
 
@@ -3103,7 +3189,7 @@ def py_call(self,
                 arg_values: List[Value],
                 line: int,
                 arg_kinds: Optional[List[int]] = None,
-                arg_names: Optional[List[Optional[str]]] = None) -> Value:
+                arg_names: Optional[Sequence[Optional[str]]] = None) -> Value:
         """Use py_call_op or py_call_with_kwargs_op for function call."""
         # If all arguments are positional, we can use py_call_op.
         if (arg_kinds is None) or all(kind == ARG_POS for kind in arg_kinds):
@@ -3152,8 +3238,8 @@ def py_method_call(self,
                        method_name: str,
                        arg_values: List[Value],
                        line: int,
-                       arg_kinds: Optional[List[int]] = None,
-                       arg_names: Optional[List[Optional[str]]] = None) -> Value:
+                       arg_kinds: Optional[List[int]],
+                       arg_names: Optional[Sequence[Optional[str]]]) -> Value:
         if (arg_kinds is None) or all(kind == ARG_POS for kind in arg_kinds):
             method_name_reg = self.load_static_unicode(method_name)
             return self.primitive_op(py_method_call_op, [obj, method_name_reg] + arg_values, line)