Various improvements to stubgen (#7921)

This improves various things in stubgen. Here are some notable changes:

* Filter out some bad type comments (or comments that look like them) since
  these generate parse errors, and stubgen can't recover from them.
* Remove the initial generated comment from all stubs, since it gets easily
  out of date and seems redundant, especially for packages with hundreds of
  submodules.
* Don't generate a blocking error for inconsistent MROs.
* Make docstring parser more robust.
* Tweak the level of logging. Add `--verbose` and `--quiet` flags.
* Export imported names that aren't references in the module.
* Fix handling of unreachable code.
* Fix signatures of various C extension methods.
* Skip modules that are not useful or can cause trouble, such as vendored 
  modules and tests.
* Preserve `@abstractproperty`.
* Fix clashes between `typing.Any` and a class named `Any` (etc.).
* Translate imports to vendored `six` to refer to normal `six`.
* Add ad-hoc whitelisting of specific definitions that should be exported.

Author: JukkaL

mypy/nodes.py

@@ -2285,6 +2285,7 @@ class is generic then it will be a type constructor of higher kind.
     # Used to stash the names of the mro classes temporarily between
     # deserialization and fixup. See deserialize() for why.
     _mro_refs = None  # type: Optional[List[str]]
+    bad_mro = False  # Could not construct full MRO
 
     declared_metaclass = None  # type: Optional[mypy.types.Instance]
     metaclass_type = None  # type: Optional[mypy.types.Instance]

mypy/options.py

@@ -3,8 +3,8 @@
 import pprint
 import sys
 
-from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple
 from typing_extensions import Final
+from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple, Callable, Any
 
 from mypy import defaults
 from mypy.util import get_class_descriptors, replace_object_state
@@ -262,6 +262,9 @@ def __init__(self) -> None:
         self.cache_map = {}  # type: Dict[str, Tuple[str, str]]
         # Don't properly free objects on exit, just kill the current process.
         self.fast_exit = False
+        # Used to transform source code before parsing if not None
+        # TODO: Make the type precise (AnyStr -> AnyStr)
+        self.transform_source = None  # type: Optional[Callable[[Any], Any]]
         # Print full path to each file in the report.
         self.show_absolute_path = False  # type: bool
 

mypy/parse.py

@@ -18,6 +18,8 @@ def parse(source: Union[str, bytes],
     The python_version (major, minor) option determines the Python syntax variant.
     """
     is_stub_file = fnam.endswith('.pyi')
+    if options.transform_source is not None:
+        source = options.transform_source(source)
     if options.python_version[0] >= 3 or is_stub_file:
         import mypy.fastparse
         return mypy.fastparse.parse(source,

mypy/semanal.py

@@ -1490,8 +1490,7 @@ def configure_base_classes(self,
 
         # Calculate the MRO.
         if not self.verify_base_classes(defn):
-            # Give it an MRO consisting of just the class itself and object.
-            defn.info.mro = [defn.info, self.object_type().type]
+            self.set_dummy_mro(defn.info)
             return
         self.calculate_class_mro(defn, self.object_type)
 
@@ -1519,6 +1518,11 @@ def configure_tuple_base_class(self,
 
         return base.partial_fallback
 
+    def set_dummy_mro(self, info: TypeInfo) -> None:
+        # Give it an MRO consisting of just the class itself and object.
+        info.mro = [info, self.object_type().type]
+        info.bad_mro = True
+
     def calculate_class_mro(self, defn: ClassDef,
                             obj_type: Optional[Callable[[], Instance]] = None) -> None:
         """Calculate method resolution order for a class.
@@ -1530,9 +1534,9 @@ def calculate_class_mro(self, defn: ClassDef,
         try:
             calculate_mro(defn.info, obj_type)
         except MroError:
-            self.fail_blocker('Cannot determine consistent method resolution '
-                              'order (MRO) for "%s"' % defn.name, defn)
-            defn.info.mro = []
+            self.fail('Cannot determine consistent method resolution '
+                      'order (MRO) for "%s"' % defn.name, defn)
+            self.set_dummy_mro(defn.info)
         # Allow plugins to alter the MRO to handle the fact that `def mro()`
         # on metaclasses permits MRO rewriting.
         if defn.fullname:
@@ -1597,12 +1601,12 @@ def update_metaclass(self, defn: ClassDef) -> None:
 
     def verify_base_classes(self, defn: ClassDef) -> bool:
         info = defn.info
+        cycle = False
         for base in info.bases:
             baseinfo = base.type
             if self.is_base_class(info, baseinfo):
-                self.fail('Cycle in inheritance hierarchy', defn, blocker=True)
-                # Clear bases to forcefully get rid of the cycle.
-                info.bases = []
+                self.fail('Cycle in inheritance hierarchy', defn)
+                cycle = True
             if baseinfo.fullname == 'builtins.bool':
                 self.fail("'%s' is not a valid base class" %
                           baseinfo.name, defn, blocker=True)
@@ -1611,7 +1615,7 @@ def verify_base_classes(self, defn: ClassDef) -> bool:
         if dup:
             self.fail('Duplicate base class "%s"' % dup.name, defn, blocker=True)
             return False
-        return True
+        return not cycle
 
     def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool:
         """Determine if t is a base class of s (but do not use mro)."""

mypy/stubdoc.py

@@ -199,8 +199,12 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[FunctionSi
     state = DocStringParser(name)
     # Return all found signatures, even if there is a parse error after some are found.
     with contextlib.suppress(tokenize.TokenError):
-        for token in tokenize.tokenize(io.BytesIO(docstr.encode('utf-8')).readline):
-            state.add_token(token)
+        try:
+            tokens = tokenize.tokenize(io.BytesIO(docstr.encode('utf-8')).readline)
+            for token in tokens:
+                state.add_token(token)
+        except IndentationError:
+            return None
     sigs = state.get_signatures()
 
     def is_unique_args(sig: FunctionSig) -> bool: