[mypyc] Irbuild docstring and comment updates, and minor refactoring (#8542)

Add and update docstrings and comments in mypyc.irbuild. Also try to
make the style of docstrings and comments more consistent and cleaner
looking. For example, generally make documentation lines shorter than
the max line length, for improved readability.

Group some related functions close to each other.

Author: JukkaL

mypyc/irbuild/callable_class.py

@@ -1,7 +1,7 @@
 """Generate a class that represents a nested function.
 
-The class defines __call__ for calling the function and allows access to variables
-defined in outer scopes.
+The class defines __call__ for calling the function and allows access to
+non-local variables defined in outer scopes.
 """
 
 from typing import List
@@ -20,21 +20,28 @@
 
 
 def setup_callable_class(builder: IRBuilder) -> None:
-    """Generates a callable class representing a nested function or a function within a
-    non-extension class and sets up the 'self' variable for that class.
+    """Generate an (incomplete) callable class representing function.
 
-    This takes the most recently visited function and returns a ClassIR to represent that
-    function. Each callable class contains an environment attribute with points to another
-    ClassIR representing the environment class where some of its variables can be accessed.
-    Note that its '__call__' method is not yet implemented, and is implemented in the
-    add_call_to_callable_class function.
+    This can be a nested function or a function within a non-extension
+    class.  Also set up the 'self' variable for that class.
 
-    Returns a newly constructed ClassIR representing the callable class for the nested
-    function.
-    """
+    This takes the most recently visited function and returns a
+    ClassIR to represent that function. Each callable class contains
+    an environment attribute which points to another ClassIR
+    representing the environment class where some of its variables can
+    be accessed.
 
-    # Check to see that the name has not already been taken. If so, rename the class. We allow
-    # multiple uses of the same function name because this is valid in if-else blocks. Example:
+    Note that some methods, such as '__call__', are not yet
+    created here. Use additional functions, such as
+    add_call_to_callable_class(), to add them.
+
+    Return a newly constructed ClassIR representing the callable
+    class for the nested function.
+    """
+    # Check to see that the name has not already been taken. If so,
+    # rename the class. We allow multiple uses of the same function
+    # name because this is valid in if-else blocks. Example:
+    #
     #     if True:
     #         def foo():          ---->    foo_obj()
     #             return True
@@ -48,12 +55,14 @@ def setup_callable_class(builder: IRBuilder) -> None:
         count += 1
     builder.callable_class_names.add(name)
 
-    # Define the actual callable class ClassIR, and set its environment to point at the
-    # previously defined environment class.
+    # Define the actual callable class ClassIR, and set its
+    # environment to point at the previously defined environment
+    # class.
     callable_class_ir = ClassIR(name, builder.module_name, is_generated=True)
 
-    # The functools @wraps decorator attempts to call setattr on nested functions, so
-    # we create a dict for these nested functions.
+    # The functools @wraps decorator attempts to call setattr on
+    # nested functions, so we create a dict for these nested
+    # functions.
     # https://github.com/python/cpython/blob/3.7/Lib/functools.py#L58
     if builder.fn_info.is_nested:
         callable_class_ir.has_dict = True
@@ -68,8 +77,8 @@ def setup_callable_class(builder: IRBuilder) -> None:
     builder.fn_info.callable_class = ImplicitClass(callable_class_ir)
     builder.classes.append(callable_class_ir)
 
-    # Add a 'self' variable to the callable class' environment, and store that variable in a
-    # register to be accessed later.
+    # Add a 'self' variable to the environment of the callable class,
+    # and store that variable in a register to be accessed later.
     self_target = add_self_to_env(builder.environment, callable_class_ir)
     builder.fn_info.callable_class.self_reg = builder.read(self_target, builder.fn_info.fitem.line)
 
@@ -79,13 +88,14 @@ def add_call_to_callable_class(builder: IRBuilder,
                                sig: FuncSignature,
                                env: Environment,
                                fn_info: FuncInfo) -> FuncIR:
-    """Generates a '__call__' method for a callable class representing a nested function.
+    """Generate a '__call__' method for a callable class representing a nested function.
 
-    This takes the blocks, signature, and environment associated with a function definition and
-    uses those to build the '__call__' method of a given callable class, used to represent that
-    function. Note that a 'self' parameter is added to its list of arguments, as the nested
-    function becomes a class method.
+    This takes the blocks, signature, and environment associated with
+    a function definition and uses those to build the '__call__'
+    method of a given callable class, used to represent that
+    function.
     """
+    # Since we create a method, we also add a 'self' parameter.
     sig = FuncSignature((RuntimeArg(SELF_NAME, object_rprimitive),) + sig.args, sig.ret_type)
     call_fn_decl = FuncDecl('__call__', fn_info.callable_class.ir.name, builder.module_name, sig)
     call_fn_ir = FuncIR(call_fn_decl, blocks, env,
@@ -95,7 +105,7 @@ def add_call_to_callable_class(builder: IRBuilder,
 
 
 def add_get_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
-    """Generates the '__get__' method for a callable class."""
+    """Generate the '__get__' method for a callable class."""
     line = fn_info.fitem.line
     builder.enter(fn_info)
 
@@ -133,22 +143,30 @@ def add_get_to_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> None:
 
 
 def instantiate_callable_class(builder: IRBuilder, fn_info: FuncInfo) -> Value:
-    """
-    Assigns a callable class to a register named after the given function definition. Note
-    that fn_info refers to the function being assigned, whereas builder.fn_info refers to the
-    function encapsulating the function being turned into a callable class.
+    """Create an instance of a callable class for a function.
+
+    Calls to the function will actually call this instance.
+
+    Note that fn_info refers to the function being assigned, whereas
+    builder.fn_info refers to the function encapsulating the function
+    being turned into a callable class.
     """
     fitem = fn_info.fitem
     func_reg = builder.add(Call(fn_info.callable_class.ir.ctor, [], fitem.line))
 
-    # Set the callable class' environment attribute to point at the environment class
-    # defined in the callable class' immediate outer scope. Note that there are three possible
-    # environment class registers we may use. If the encapsulating function is:
-    # - a generator function, then the callable class is instantiated from the generator class'
-    #   __next__' function, and hence the generator class' environment register is used.
-    # - a nested function, then the callable class is instantiated from the current callable
-    #   class' '__call__' function, and hence the callable class' environment register is used.
-    # - neither, then we use the environment register of the original function.
+    # Set the environment attribute of the callable class to point at
+    # the environment class defined in the callable class' immediate
+    # outer scope. Note that there are three possible environment
+    # class registers we may use. This depends on what the encapsulating
+    # (parent) function is:
+    #
+    # - A nested function: the callable class is instantiated
+    #   from the current callable class' '__call__' function, and hence
+    #   the callable class' environment register is used.
+    # - A generator function: the callable class is instantiated
+    #   from the '__next__' method of the generator class, and hence the
+    #   environment of the generator class is used.
+    # - Regular function: we use the environment of the original function.
     curr_env_reg = None
     if builder.fn_info.is_generator:
         curr_env_reg = builder.fn_info.generator_class.curr_env_reg

mypyc/irbuild/classdef.py

@@ -1,3 +1,5 @@
+"""Transform class definitions from the mypy AST form to IR."""
+
 from typing import List, Optional
 
 from mypy.nodes import (
@@ -29,6 +31,17 @@
 
 
 def transform_class_def(builder: IRBuilder, cdef: ClassDef) -> None:
+    """Create IR for a class definition.
+
+    This can generate both extension (native) and non-extension
+    classes.  These are generated in very different ways. In the
+    latter case we construct a Python type object at runtime by doing
+    the equivalent of "type(name, bases, dict)" in IR. Extension
+    classes are defined via C structs that are generated later in
+    mypyc.codegen.emitclass.
+
+    This is the main entry point to this module.
+    """
     ir = builder.mapper.type_to_ir[cdef.info]
 
     # We do this check here because the base field of parent
@@ -188,9 +201,9 @@ def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
 
 
 def populate_non_ext_bases(builder: IRBuilder, cdef: ClassDef) -> Value:
-    """
-    Populate the base-class tuple passed to the metaclass constructor
-    for non-extension classes.
+    """Create base class tuple of a non-extension class.
+
+    The tuple is passed to the metaclass constructor.
     """
     ir = builder.mapper.type_to_ir[cdef.info]
     bases = []
@@ -222,11 +235,10 @@ def setup_non_ext_dict(builder: IRBuilder,
                        cdef: ClassDef,
                        metaclass: Value,
                        bases: Value) -> Value:
-    """
-    Initialize the class dictionary for a non-extension class. This class dictionary
-    is passed to the metaclass constructor.
-    """
+    """Initialize the class dictionary for a non-extension class.
 
+    This class dictionary is passed to the metaclass constructor.
+    """
     # Check if the metaclass defines a __prepare__ method, and if so, call it.
     has_prepare = builder.primitive_op(py_hasattr_op,
                                     [metaclass,
@@ -252,14 +264,16 @@ def setup_non_ext_dict(builder: IRBuilder,
     return non_ext_dict
 
 
-def add_non_ext_class_attr(builder: IRBuilder, non_ext: NonExtClassInfo, lvalue: NameExpr,
-                           stmt: AssignmentStmt, cdef: ClassDef,
+def add_non_ext_class_attr(builder: IRBuilder,
+                           non_ext: NonExtClassInfo,
+                           lvalue: NameExpr,
+                           stmt: AssignmentStmt,
+                           cdef: ClassDef,
                            attr_to_cache: List[Lvalue]) -> None:
-    """
-    Add a class attribute to __annotations__ of a non-extension class. If the
-    attribute is assigned to a value, it is also added to __dict__.
-    """
+    """Add a class attribute to __annotations__ of a non-extension class.
 
+    If the attribute is initialized with a value, also add it to __dict__.
+    """
     # We populate __annotations__ because dataclasses uses it to determine
     # which attributes to compute on.
     # TODO: Maybe generate more precise types for annotations
@@ -284,7 +298,7 @@ def add_non_ext_class_attr(builder: IRBuilder, non_ext: NonExtClassInfo, lvalue:
 
 
 def generate_attr_defaults(builder: IRBuilder, cdef: ClassDef) -> None:
-    """Generate an initialization method for default attr values (from class vars)"""
+    """Generate an initialization method for default attr values (from class vars)."""
     cls = builder.mapper.type_to_ir[cdef.info]
     if cls.builtin_base:
         return
@@ -347,6 +361,7 @@ def generate_attr_defaults(builder: IRBuilder, cdef: ClassDef) -> None:
 
 
 def create_ne_from_eq(builder: IRBuilder, cdef: ClassDef) -> None:
+    """Create a "__ne__" method from a "__eq__" method (if only latter exists)."""
     cls = builder.mapper.type_to_ir[cdef.info]
     if cls.has_method('__eq__') and not cls.has_method('__ne__'):
         f = gen_glue_ne_method(builder, cls, cdef.line)
@@ -356,7 +371,7 @@ def create_ne_from_eq(builder: IRBuilder, cdef: ClassDef) -> None:
 
 
 def gen_glue_ne_method(builder: IRBuilder, cls: ClassIR, line: int) -> FuncIR:
-    """Generate a __ne__ method from a __eq__ method. """
+    """Generate a "__ne__" method from a "__eq__" method. """
     builder.enter()
 
     rt_args = (RuntimeArg("self", RInstance(cls)), RuntimeArg("rhs", object_rprimitive))
@@ -417,10 +432,13 @@ def load_non_ext_class(builder: IRBuilder,
 
 
 def load_decorated_class(builder: IRBuilder, cdef: ClassDef, type_obj: Value) -> Value:
-    """
-    Given a decorated ClassDef and a register containing a non-extension representation of the
-    ClassDef created via the type constructor, applies the corresponding decorator functions
-    on that decorated ClassDef and returns a register containing the decorated ClassDef.
+    """Apply class decorators to create a decorated (non-extension) class object.
+
+    Given a decorated ClassDef and a register containing a
+    non-extension representation of the ClassDef created via the type
+    constructor, applies the corresponding decorator functions on that
+    decorated ClassDef and returns a register containing the decorated
+    ClassDef.
     """
     decorators = cdef.decorators
     dec_class = type_obj
@@ -432,7 +450,7 @@ def load_decorated_class(builder: IRBuilder, cdef: ClassDef, type_obj: Value) ->
 
 
 def cache_class_attrs(builder: IRBuilder, attrs_to_cache: List[Lvalue], cdef: ClassDef) -> None:
-    """Add class attributes to be cached to the global cache"""
+    """Add class attributes to be cached to the global cache."""
     typ = builder.load_native_type_object(cdef.fullname)
     for lval in attrs_to_cache:
         assert isinstance(lval, NameExpr)

mypyc/irbuild/context.py

@@ -1,3 +1,5 @@
+"""Helpers that store information about functions and the related classes."""
+
 from typing import List, Optional, Tuple
 
 from mypy.nodes import FuncItem
@@ -10,6 +12,7 @@
 
 class FuncInfo:
     """Contains information about functions as they are generated."""
+
     def __init__(self,
                  fitem: FuncItem = INVALID_FUNC_DEF,
                  name: str = '',
@@ -87,9 +90,14 @@ def curr_env_reg(self) -> Value:
 
 
 class ImplicitClass:
-    """Contains information regarding classes that are generated as a result of nested functions or
-    generated functions, but not explicitly defined in the source code.
+    """Contains information regarding implicitly generated classes.
+
+    Implicit classes are generated for nested functions and generator
+    functions. They are not explicitly defined in the source code.
+
+    NOTE: This is both a concrete class and used as a base class.
     """
+
     def __init__(self, ir: ClassIR) -> None:
         # The ClassIR instance associated with this class.
         self.ir = ir
@@ -131,6 +139,8 @@ def prev_env_reg(self, reg: Value) -> None:
 
 
 class GeneratorClass(ImplicitClass):
+    """Contains information about implicit generator function classes."""
+
     def __init__(self, ir: ClassIR) -> None:
         super().__init__(ir)
         # This register holds the label number that the '__next__' function should go to the next