From 25776360f389f5f446319c92531b695826df902d Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 14:35:34 -0800
Subject: [PATCH 01/74] stubtest: don't hardcode python version

This will cause us to assume python version is sys.version_info, which
is the behaviour we want, since we're comparing things to the runtime
---
 scripts/stubtest.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 048075f1445e..2c31bb342ca1 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -218,7 +218,6 @@ def main(args: List[str]) -> Iterator[Error]:
         modules = args[1:]
 
     options = Options()
-    options.incremental = False
     data_dir = default_data_dir()
     search_path = compute_search_paths([], options, data_dir)
     find_module_cache = FindModuleCache(search_path)

From 2d0438572ed94afb5afc015aaba09572ca0835e7 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 14:36:53 -0800
Subject: [PATCH 02/74] stubtest: fix build

---
 scripts/stubtest.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 2c31bb342ca1..048075f1445e 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -218,6 +218,7 @@ def main(args: List[str]) -> Iterator[Error]:
         modules = args[1:]
 
     options = Options()
+    options.incremental = False
     data_dir = default_data_dir()
     search_path = compute_search_paths([], options, data_dir)
     find_module_cache = FindModuleCache(search_path)

From 908ef642fc8ce2e542ec39c9f1511ee4389c4aff Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 15:09:30 -0800
Subject: [PATCH 03/74] stubtest: recognise typealias

---
 scripts/stubtest.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 048075f1445e..99f2fc448692 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -188,6 +188,13 @@ def verify_typealias(node: nodes.TypeAlias,
         yield None
 
 
+@verify.register(nodes.TypeAlias)
+def verify_typealias(node: nodes.TypeAlias,
+                     module_node: Optional[DumpNode]) -> Iterator[ErrorParts]:
+    if False:
+        yield None
+
+
 def dump_module(name: str) -> DumpNode:
     mod = importlib.import_module(name)
     return {'type': 'file', 'names': module_to_json(mod)}

From 33b98cdfc40561e962defb99d4b638e858ec398b Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 19:58:25 -0800
Subject: [PATCH 04/74] stubtest: use argparse, support custom typeshed dir

---
 scripts/stubtest.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 99f2fc448692..9626233ab3fb 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -4,6 +4,7 @@
 at runtime.
 """
 
+import argparse
 import importlib
 import sys
 from typing import Dict, Any, List, Iterator, NamedTuple, Optional, Mapping, Tuple
@@ -217,25 +218,27 @@ def build_stubs(options: Options,
     return res.files
 
 
-def main(args: List[str]) -> Iterator[Error]:
-    if len(args) == 1:
-        print('must provide at least one module to test')
-        sys.exit(1)
-    else:
-        modules = args[1:]
+def main() -> Iterator[Error]:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('modules', nargs='+', help='Modules to test')
+    parser.add_argument(
+        '--custom-typeshed-dir', metavar='DIR', help='Use the custom typeshed in DIR'
+    )
+    args = parser.parse_args()
 
     options = Options()
     options.incremental = False
+    options.custom_typeshed_dir = args.custom_typeshed_dir
+
     data_dir = default_data_dir()
     search_path = compute_search_paths([], options, data_dir)
     find_module_cache = FindModuleCache(search_path)
 
-    for module in modules:
+    for module in args.modules:
         for error in test_stub(options, find_module_cache, module):
             yield error
 
 
 if __name__ == '__main__':
-
-    for err in main(sys.argv):
+    for err in main():
         print(messages[err.error_type].format(error=err))

From 3161d43ed499b057ee16e529b27186c7445da650 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 20:03:34 -0800
Subject: [PATCH 05/74] stubtest: [minor] blacken

---
 scripts/stubtest.py | 191 ++++++++++++++++++++++++--------------------
 1 file changed, 106 insertions(+), 85 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 9626233ab3fb..32685d8391bf 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -26,79 +26,90 @@
 #       This seems like it should not be.
 
 skip = {
-    '_importlib_modulespec',
-    '_subprocess',
-    'distutils.command.bdist_msi',
-    'distutils.command.bdist_packager',
-    'msvcrt',
-    'wsgiref.types',
-    'mypy_extensions',
-    'unittest.mock',  # mock.call infinite loops on inspect.getsourcelines
-                      # https://bugs.python.org/issue25532
-                      # TODO: can we filter only call?
+    "_importlib_modulespec",
+    "_subprocess",
+    "distutils.command.bdist_msi",
+    "distutils.command.bdist_packager",
+    "msvcrt",
+    "wsgiref.types",
+    "mypy_extensions",
+    "unittest.mock",  # mock.call infinite loops on inspect.getsourcelines
+    # https://bugs.python.org/issue25532
+    # TODO: can we filter only call?
 }  # type: Final
 
 
 messages = {
-    'not_in_runtime': ('{error.stub_type} "{error.name}" defined at line '
-                       ' {error.line} in stub but is not defined at runtime'),
-    'not_in_stub': ('{error.module_type} "{error.name}" defined at line'
-                    ' {error.line} at runtime but is not defined in stub'),
-    'no_stubs': 'could not find typeshed {error.name}',
-    'inconsistent': ('"{error.name}" is {error.stub_type} in stub but'
-                     ' {error.module_type} at runtime'),
+    "not_in_runtime": (
+        '{error.stub_type} "{error.name}" defined at line '
+        " {error.line} in stub but is not defined at runtime"
+    ),
+    "not_in_stub": (
+        '{error.module_type} "{error.name}" defined at line'
+        " {error.line} at runtime but is not defined in stub"
+    ),
+    "no_stubs": "could not find typeshed {error.name}",
+    "inconsistent": (
+        '"{error.name}" is {error.stub_type} in stub but'
+        " {error.module_type} at runtime"
+    ),
 }  # type: Final
 
-Error = NamedTuple('Error', (
-    ('module', str),
-    ('name', str),
-    ('error_type', str),
-    ('line', Optional[int]),
-    ('stub_type', Optional[Type[nodes.Node]]),
-    ('module_type', Optional[str]),
-))
+Error = NamedTuple(
+    "Error",
+    (
+        ("module", str),
+        ("name", str),
+        ("error_type", str),
+        ("line", Optional[int]),
+        ("stub_type", Optional[Type[nodes.Node]]),
+        ("module_type", Optional[str]),
+    ),
+)
 
 ErrorParts = Tuple[
-    List[str],
-    str,
-    Optional[int],
-    Optional[Type[nodes.Node]],
-    Optional[str],
+    List[str], str, Optional[int], Optional[Type[nodes.Node]], Optional[str],
 ]
 
 
-def test_stub(options: Options,
-              find_module_cache: FindModuleCache,
-              name: str) -> Iterator[Error]:
+def test_stub(
+    options: Options, find_module_cache: FindModuleCache, name: str
+) -> Iterator[Error]:
     stubs = {
-        mod: stub for mod, stub in build_stubs(options, find_module_cache, name).items()
-        if (mod == name or mod.startswith(name + '.')) and mod not in skip
+        mod: stub
+        for mod, stub in build_stubs(options, find_module_cache, name).items()
+        if (mod == name or mod.startswith(name + ".")) and mod not in skip
     }
 
     for mod, stub in stubs.items():
         instance = dump_module(mod)
 
-        for identifiers, error_type, line, stub_type, module_type in verify(stub, instance):
-            yield Error(mod, '.'.join(identifiers), error_type, line, stub_type, module_type)
+        for identifiers, error_type, line, stub_type, module_type in verify(
+            stub, instance
+        ):
+            yield Error(
+                mod, ".".join(identifiers), error_type, line, stub_type, module_type
+            )
 
 
 @singledispatch
-def verify(node: nodes.Node,
-           module_node: Optional[DumpNode]) -> Iterator[ErrorParts]:
-    raise TypeError('unknown mypy node ' + str(node))
-
+def verify(node: nodes.Node, module_node: Optional[DumpNode]) -> Iterator[ErrorParts]:
+    raise TypeError("unknown mypy node " + str(node))
 
 
 @verify.register(nodes.MypyFile)
-def verify_mypyfile(stub: nodes.MypyFile,
-                    instance: Optional[DumpNode]) -> Iterator[ErrorParts]:
+def verify_mypyfile(
+    stub: nodes.MypyFile, instance: Optional[DumpNode]
+) -> Iterator[ErrorParts]:
     if instance is None:
-        yield [], 'not_in_runtime', stub.line, type(stub), None
-    elif instance['type'] != 'file':
-        yield [], 'inconsistent', stub.line, type(stub), instance['type']
+        yield [], "not_in_runtime", stub.line, type(stub), None
+    elif instance["type"] != "file":
+        yield [], "inconsistent", stub.line, type(stub), instance["type"]
     else:
-        stub_children = defaultdict(lambda: None, stub.names)  # type: Mapping[str, Optional[nodes.SymbolTableNode]]
-        instance_children = defaultdict(lambda: None, instance['names'])
+        stub_children = defaultdict(
+            lambda: None, stub.names
+        )  # type: Mapping[str, Optional[nodes.SymbolTableNode]]
+        instance_children = defaultdict(lambda: None, instance["names"])
 
         # TODO: I would rather not filter public children here.
         #       For example, what if the checkersurfaces an inconsistency
@@ -106,85 +117,95 @@ def verify_mypyfile(stub: nodes.MypyFile,
         public_nodes = {
             name: (stub_children[name], instance_children[name])
             for name in set(stub_children) | set(instance_children)
-            if not name.startswith('_')
+            if not name.startswith("_")
             and (stub_children[name] is None or stub_children[name].module_public)  # type: ignore
         }
 
         for node, (stub_child, instance_child) in public_nodes.items():
-            stub_child = getattr(stub_child, 'node', None)
-            for identifiers, error_type, line, stub_type, module_type in verify(stub_child, instance_child):
+            stub_child = getattr(stub_child, "node", None)
+            for identifiers, error_type, line, stub_type, module_type in verify(
+                stub_child, instance_child
+            ):
                 yield ([node] + identifiers, error_type, line, stub_type, module_type)
 
 
 @verify.register(nodes.TypeInfo)
-def verify_typeinfo(stub: nodes.TypeInfo,
-                    instance: Optional[DumpNode]) -> Iterator[ErrorParts]:
+def verify_typeinfo(
+    stub: nodes.TypeInfo, instance: Optional[DumpNode]
+) -> Iterator[ErrorParts]:
     if not instance:
-        yield [], 'not_in_runtime', stub.line, type(stub), None
-    elif instance['type'] != 'class':
-        yield [], 'inconsistent', stub.line, type(stub), instance['type']
+        yield [], "not_in_runtime", stub.line, type(stub), None
+    elif instance["type"] != "class":
+        yield [], "inconsistent", stub.line, type(stub), instance["type"]
     else:
         for attr, attr_node in stub.names.items():
-            subdump = instance['attributes'].get(attr, None)
-            for identifiers, error_type, line, stub_type, module_type in verify(attr_node.node, subdump):
+            subdump = instance["attributes"].get(attr, None)
+            for identifiers, error_type, line, stub_type, module_type in verify(
+                attr_node.node, subdump
+            ):
                 yield ([attr] + identifiers, error_type, line, stub_type, module_type)
 
 
 @verify.register(nodes.FuncItem)
-def verify_funcitem(stub: nodes.FuncItem,
-                    instance: Optional[DumpNode]) -> Iterator[ErrorParts]:
+def verify_funcitem(
+    stub: nodes.FuncItem, instance: Optional[DumpNode]
+) -> Iterator[ErrorParts]:
     if not instance:
-        yield [], 'not_in_runtime', stub.line, type(stub), None
-    elif 'type' not in instance or instance['type'] not in ('function', 'callable'):
-        yield [], 'inconsistent', stub.line, type(stub), instance['type']
+        yield [], "not_in_runtime", stub.line, type(stub), None
+    elif "type" not in instance or instance["type"] not in ("function", "callable"):
+        yield [], "inconsistent", stub.line, type(stub), instance["type"]
     # TODO check arguments and return value
 
 
 @verify.register(type(None))
-def verify_none(stub: None,
-                instance: Optional[DumpNode]) -> Iterator[ErrorParts]:
+def verify_none(stub: None, instance: Optional[DumpNode]) -> Iterator[ErrorParts]:
     if instance is None:
-        yield [], 'not_in_stub', None, None, None
+        yield [], "not_in_stub", None, None, None
     else:
-        yield [], 'not_in_stub', instance['line'], None, instance['type']
+        yield [], "not_in_stub", instance["line"], None, instance["type"]
 
 
 @verify.register(nodes.Var)
-def verify_var(node: nodes.Var,
-               module_node: Optional[DumpNode]) -> Iterator[ErrorParts]:
+def verify_var(
+    node: nodes.Var, module_node: Optional[DumpNode]
+) -> Iterator[ErrorParts]:
     if False:
         yield None
     # Need to check if types are inconsistent.
-    #if 'type' not in dump or dump['type'] != node.node.type:
+    # if 'type' not in dump or dump['type'] != node.node.type:
     #    import ipdb; ipdb.set_trace()
     #    yield name, 'inconsistent', node.node.line, shed_type, module_type
 
 
 @verify.register(nodes.OverloadedFuncDef)
-def verify_overloadedfuncdef(node: nodes.OverloadedFuncDef,
-                             module_node: Optional[DumpNode]) -> Iterator[ErrorParts]:
+def verify_overloadedfuncdef(
+    node: nodes.OverloadedFuncDef, module_node: Optional[DumpNode]
+) -> Iterator[ErrorParts]:
     # Should check types of the union of the overloaded types.
     if False:
         yield None
 
 
 @verify.register(nodes.TypeVarExpr)
-def verify_typevarexpr(node: nodes.TypeVarExpr,
-                       module_node: Optional[DumpNode]) -> Iterator[ErrorParts]:
+def verify_typevarexpr(
+    node: nodes.TypeVarExpr, module_node: Optional[DumpNode]
+) -> Iterator[ErrorParts]:
     if False:
         yield None
 
 
 @verify.register(nodes.Decorator)
-def verify_decorator(node: nodes.Decorator,
-                     module_node: Optional[DumpNode]) -> Iterator[ErrorParts]:
+def verify_decorator(
+    node: nodes.Decorator, module_node: Optional[DumpNode]
+) -> Iterator[ErrorParts]:
     if False:
         yield None
 
 
 @verify.register(nodes.TypeAlias)
-def verify_typealias(node: nodes.TypeAlias,
-                     module_node: Optional[DumpNode]) -> Iterator[ErrorParts]:
+def verify_typealias(
+    node: nodes.TypeAlias, module_node: Optional[DumpNode]
+) -> Iterator[ErrorParts]:
     if False:
         yield None
 
@@ -198,12 +219,12 @@ def verify_typealias(node: nodes.TypeAlias,
 
 def dump_module(name: str) -> DumpNode:
     mod = importlib.import_module(name)
-    return {'type': 'file', 'names': module_to_json(mod)}
+    return {"type": "file", "names": module_to_json(mod)}
 
 
-def build_stubs(options: Options,
-                find_module_cache: FindModuleCache,
-                mod: str) -> Dict[str, nodes.MypyFile]:
+def build_stubs(
+    options: Options, find_module_cache: FindModuleCache, mod: str
+) -> Dict[str, nodes.MypyFile]:
     sources = find_module_cache.find_modules_recursive(mod)
     try:
         res = build.build(sources=sources, options=options)
@@ -220,9 +241,9 @@ def build_stubs(options: Options,
 
 def main() -> Iterator[Error]:
     parser = argparse.ArgumentParser()
-    parser.add_argument('modules', nargs='+', help='Modules to test')
+    parser.add_argument("modules", nargs="+", help="Modules to test")
     parser.add_argument(
-        '--custom-typeshed-dir', metavar='DIR', help='Use the custom typeshed in DIR'
+        "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
     )
     args = parser.parse_args()
 
@@ -239,6 +260,6 @@ def main() -> Iterator[Error]:
             yield error
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     for err in main():
         print(messages[err.error_type].format(error=err))

From 03d223de6998bb6bb609f5ca81868e8ca32381c8 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 20:05:31 -0800
Subject: [PATCH 06/74] stubtest: [minor] import nits

---
 scripts/stubtest.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 32685d8391bf..71bd4d5f58ce 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -12,9 +12,9 @@
 from collections import defaultdict
 from functools import singledispatch
 
-from mypy import build
-from mypy.build import default_data_dir
-from mypy.modulefinder import compute_search_paths, FindModuleCache
+import mypy.build
+import mypy.modulefinder
+from mypy.modulefinder import FindModuleCache
 from mypy.errors import CompileError
 from mypy import nodes
 from mypy.options import Options
@@ -227,7 +227,7 @@ def build_stubs(
 ) -> Dict[str, nodes.MypyFile]:
     sources = find_module_cache.find_modules_recursive(mod)
     try:
-        res = build.build(sources=sources, options=options)
+        res = mypy.build.build(sources=sources, options=options)
         messages = res.errors
     except CompileError as error:
         messages = error.messages
@@ -251,8 +251,8 @@ def main() -> Iterator[Error]:
     options.incremental = False
     options.custom_typeshed_dir = args.custom_typeshed_dir
 
-    data_dir = default_data_dir()
-    search_path = compute_search_paths([], options, data_dir)
+    data_dir = mypy.build.default_data_dir()
+    search_path = mypy.modulefinder.compute_search_paths([], options, data_dir)
     find_module_cache = FindModuleCache(search_path)
 
     for module in args.modules:

From 8036364065a0612ed4d021c95c0afbf7bd00d962 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 20:09:52 -0800
Subject: [PATCH 07/74] stubtest: [minor] renames, reorder parameters

---
 scripts/stubtest.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 71bd4d5f58ce..5dc951d13f74 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -25,7 +25,7 @@
 # TODO: email.contentmanager has a symbol table with a None node.
 #       This seems like it should not be.
 
-skip = {
+MODULES_TO_SKIP = {
     "_importlib_modulespec",
     "_subprocess",
     "distutils.command.bdist_msi",
@@ -72,13 +72,14 @@
 ]
 
 
-def test_stub(
-    options: Options, find_module_cache: FindModuleCache, name: str
+def test_module(
+    module_name: str, options: Options, find_module_cache: FindModuleCache
 ) -> Iterator[Error]:
     stubs = {
         mod: stub
-        for mod, stub in build_stubs(options, find_module_cache, name).items()
-        if (mod == name or mod.startswith(name + ".")) and mod not in skip
+        for mod, stub in build_stubs(module_name, options, find_module_cache).items()
+        if (mod == module_name or mod.startswith(module_name + "."))
+        and mod not in MODULES_TO_SKIP
     }
 
     for mod, stub in stubs.items():
@@ -223,9 +224,9 @@ def dump_module(name: str) -> DumpNode:
 
 
 def build_stubs(
-    options: Options, find_module_cache: FindModuleCache, mod: str
+    module_name: str, options: Options, find_module_cache: FindModuleCache
 ) -> Dict[str, nodes.MypyFile]:
-    sources = find_module_cache.find_modules_recursive(mod)
+    sources = find_module_cache.find_modules_recursive(module_name)
     try:
         res = mypy.build.build(sources=sources, options=options)
         messages = res.errors
@@ -256,7 +257,7 @@ def main() -> Iterator[Error]:
     find_module_cache = FindModuleCache(search_path)
 
     for module in args.modules:
-        for error in test_stub(options, find_module_cache, module):
+        for error in test_module(module, options, find_module_cache):
             yield error
 
 

From e861ff9b10dd9840e2e88b66da39a61b89ce60d3 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 21:02:32 -0800
Subject: [PATCH 08/74] stubtest: [wip] start to use runtime objects directly

---
 scripts/stubtest.py | 48 ++++++++++++---------------------------------
 1 file changed, 13 insertions(+), 35 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 5dc951d13f74..335786d80a1b 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -19,8 +19,6 @@
 from mypy import nodes
 from mypy.options import Options
 
-from dumpmodule import module_to_json, DumpNode
-
 
 # TODO: email.contentmanager has a symbol table with a None node.
 #       This seems like it should not be.
@@ -83,24 +81,18 @@ def test_module(
     }
 
     for mod, stub in stubs.items():
-        instance = dump_module(mod)
-
-        for identifiers, error_type, line, stub_type, module_type in verify(
-            stub, instance
-        ):
-            yield Error(
-                mod, ".".join(identifiers), error_type, line, stub_type, module_type
-            )
+        runtime = importlib.import_module(mod)
+        yield from verify(stub, runtime)
 
 
 @singledispatch
-def verify(node: nodes.Node, module_node: Optional[DumpNode]) -> Iterator[ErrorParts]:
-    raise TypeError("unknown mypy node " + str(node))
+def verify(stub: nodes.Node, runtime: Optional[Any]) -> Iterator[ErrorParts]:
+    raise TypeError("unknown mypy node " + str(stub))
 
 
 @verify.register(nodes.MypyFile)
 def verify_mypyfile(
-    stub: nodes.MypyFile, instance: Optional[DumpNode]
+    stub: nodes.MypyFile, instance: Optional[Any]
 ) -> Iterator[ErrorParts]:
     if instance is None:
         yield [], "not_in_runtime", stub.line, type(stub), None
@@ -132,7 +124,7 @@ def verify_mypyfile(
 
 @verify.register(nodes.TypeInfo)
 def verify_typeinfo(
-    stub: nodes.TypeInfo, instance: Optional[DumpNode]
+    stub: nodes.TypeInfo, instance: Optional[Any]
 ) -> Iterator[ErrorParts]:
     if not instance:
         yield [], "not_in_runtime", stub.line, type(stub), None
@@ -149,7 +141,7 @@ def verify_typeinfo(
 
 @verify.register(nodes.FuncItem)
 def verify_funcitem(
-    stub: nodes.FuncItem, instance: Optional[DumpNode]
+    stub: nodes.FuncItem, instance: Optional[Any]
 ) -> Iterator[ErrorParts]:
     if not instance:
         yield [], "not_in_runtime", stub.line, type(stub), None
@@ -159,7 +151,7 @@ def verify_funcitem(
 
 
 @verify.register(type(None))
-def verify_none(stub: None, instance: Optional[DumpNode]) -> Iterator[ErrorParts]:
+def verify_none(stub: None, instance: Optional[Any]) -> Iterator[ErrorParts]:
     if instance is None:
         yield [], "not_in_stub", None, None, None
     else:
@@ -167,9 +159,7 @@ def verify_none(stub: None, instance: Optional[DumpNode]) -> Iterator[ErrorParts
 
 
 @verify.register(nodes.Var)
-def verify_var(
-    node: nodes.Var, module_node: Optional[DumpNode]
-) -> Iterator[ErrorParts]:
+def verify_var(node: nodes.Var, module_node: Optional[Any]) -> Iterator[ErrorParts]:
     if False:
         yield None
     # Need to check if types are inconsistent.
@@ -180,7 +170,7 @@ def verify_var(
 
 @verify.register(nodes.OverloadedFuncDef)
 def verify_overloadedfuncdef(
-    node: nodes.OverloadedFuncDef, module_node: Optional[DumpNode]
+    node: nodes.OverloadedFuncDef, module_node: Optional[Any]
 ) -> Iterator[ErrorParts]:
     # Should check types of the union of the overloaded types.
     if False:
@@ -189,7 +179,7 @@ def verify_overloadedfuncdef(
 
 @verify.register(nodes.TypeVarExpr)
 def verify_typevarexpr(
-    node: nodes.TypeVarExpr, module_node: Optional[DumpNode]
+    node: nodes.TypeVarExpr, module_node: Optional[Any]
 ) -> Iterator[ErrorParts]:
     if False:
         yield None
@@ -197,7 +187,7 @@ def verify_typevarexpr(
 
 @verify.register(nodes.Decorator)
 def verify_decorator(
-    node: nodes.Decorator, module_node: Optional[DumpNode]
+    node: nodes.Decorator, module_node: Optional[Any]
 ) -> Iterator[ErrorParts]:
     if False:
         yield None
@@ -205,24 +195,12 @@ def verify_decorator(
 
 @verify.register(nodes.TypeAlias)
 def verify_typealias(
-    node: nodes.TypeAlias, module_node: Optional[DumpNode]
+    node: nodes.TypeAlias, module_node: Optional[Any]
 ) -> Iterator[ErrorParts]:
     if False:
         yield None
 
 
-@verify.register(nodes.TypeAlias)
-def verify_typealias(node: nodes.TypeAlias,
-                     module_node: Optional[DumpNode]) -> Iterator[ErrorParts]:
-    if False:
-        yield None
-
-
-def dump_module(name: str) -> DumpNode:
-    mod = importlib.import_module(name)
-    return {"type": "file", "names": module_to_json(mod)}
-
-
 def build_stubs(
     module_name: str, options: Options, find_module_cache: FindModuleCache
 ) -> Dict[str, nodes.MypyFile]:

From 3651e6f305a5766b4e307b942da2743cbd263a19 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 21:03:38 -0800
Subject: [PATCH 09/74] stubtest: [minor] make parameter names for verify_*
 consistent

---
 scripts/stubtest.py | 42 +++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 335786d80a1b..1373cefba6a4 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -92,47 +92,47 @@ def verify(stub: nodes.Node, runtime: Optional[Any]) -> Iterator[ErrorParts]:
 
 @verify.register(nodes.MypyFile)
 def verify_mypyfile(
-    stub: nodes.MypyFile, instance: Optional[Any]
+    stub: nodes.MypyFile, runtime: Optional[Any]
 ) -> Iterator[ErrorParts]:
-    if instance is None:
+    if runtime is None:
         yield [], "not_in_runtime", stub.line, type(stub), None
-    elif instance["type"] != "file":
-        yield [], "inconsistent", stub.line, type(stub), instance["type"]
+    elif runtime["type"] != "file":
+        yield [], "inconsistent", stub.line, type(stub), runtime["type"]
     else:
         stub_children = defaultdict(
             lambda: None, stub.names
         )  # type: Mapping[str, Optional[nodes.SymbolTableNode]]
-        instance_children = defaultdict(lambda: None, instance["names"])
+        runtime_children = defaultdict(lambda: None, runtime["names"])
 
         # TODO: I would rather not filter public children here.
         #       For example, what if the checkersurfaces an inconsistency
         #       in the typing of a private child
         public_nodes = {
-            name: (stub_children[name], instance_children[name])
-            for name in set(stub_children) | set(instance_children)
+            name: (stub_children[name], runtime_children[name])
+            for name in set(stub_children) | set(runtime_children)
             if not name.startswith("_")
             and (stub_children[name] is None or stub_children[name].module_public)  # type: ignore
         }
 
-        for node, (stub_child, instance_child) in public_nodes.items():
+        for node, (stub_child, runtime_child) in public_nodes.items():
             stub_child = getattr(stub_child, "node", None)
             for identifiers, error_type, line, stub_type, module_type in verify(
-                stub_child, instance_child
+                stub_child, runtime_child
             ):
                 yield ([node] + identifiers, error_type, line, stub_type, module_type)
 
 
 @verify.register(nodes.TypeInfo)
 def verify_typeinfo(
-    stub: nodes.TypeInfo, instance: Optional[Any]
+    stub: nodes.TypeInfo, runtime: Optional[Any]
 ) -> Iterator[ErrorParts]:
-    if not instance:
+    if not runtime:
         yield [], "not_in_runtime", stub.line, type(stub), None
-    elif instance["type"] != "class":
-        yield [], "inconsistent", stub.line, type(stub), instance["type"]
+    elif runtime["type"] != "class":
+        yield [], "inconsistent", stub.line, type(stub), runtime["type"]
     else:
         for attr, attr_node in stub.names.items():
-            subdump = instance["attributes"].get(attr, None)
+            subdump = runtime["attributes"].get(attr, None)
             for identifiers, error_type, line, stub_type, module_type in verify(
                 attr_node.node, subdump
             ):
@@ -141,21 +141,21 @@ def verify_typeinfo(
 
 @verify.register(nodes.FuncItem)
 def verify_funcitem(
-    stub: nodes.FuncItem, instance: Optional[Any]
+    stub: nodes.FuncItem, runtime: Optional[Any]
 ) -> Iterator[ErrorParts]:
-    if not instance:
+    if not runtime:
         yield [], "not_in_runtime", stub.line, type(stub), None
-    elif "type" not in instance or instance["type"] not in ("function", "callable"):
-        yield [], "inconsistent", stub.line, type(stub), instance["type"]
+    elif "type" not in runtime or runtime["type"] not in ("function", "callable"):
+        yield [], "inconsistent", stub.line, type(stub), runtime["type"]
     # TODO check arguments and return value
 
 
 @verify.register(type(None))
-def verify_none(stub: None, instance: Optional[Any]) -> Iterator[ErrorParts]:
-    if instance is None:
+def verify_none(stub: None, runtime: Optional[Any]) -> Iterator[ErrorParts]:
+    if runtime is None:
         yield [], "not_in_stub", None, None, None
     else:
-        yield [], "not_in_stub", instance["line"], None, instance["type"]
+        yield [], "not_in_stub", runtime["line"], None, runtime["type"]
 
 
 @verify.register(nodes.Var)

From ad2a7c8424a11cf5c6936133a866ce1f199d6272 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 21:08:45 -0800
Subject: [PATCH 10/74] stubtest: gut error handling

---
 scripts/stubtest.py | 120 +++++++++++---------------------------------
 1 file changed, 28 insertions(+), 92 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 1373cefba6a4..dda2dbb0eb41 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -20,54 +20,8 @@
 from mypy.options import Options
 
 
-# TODO: email.contentmanager has a symbol table with a None node.
-#       This seems like it should not be.
-
-MODULES_TO_SKIP = {
-    "_importlib_modulespec",
-    "_subprocess",
-    "distutils.command.bdist_msi",
-    "distutils.command.bdist_packager",
-    "msvcrt",
-    "wsgiref.types",
-    "mypy_extensions",
-    "unittest.mock",  # mock.call infinite loops on inspect.getsourcelines
-    # https://bugs.python.org/issue25532
-    # TODO: can we filter only call?
-}  # type: Final
-
-
-messages = {
-    "not_in_runtime": (
-        '{error.stub_type} "{error.name}" defined at line '
-        " {error.line} in stub but is not defined at runtime"
-    ),
-    "not_in_stub": (
-        '{error.module_type} "{error.name}" defined at line'
-        " {error.line} at runtime but is not defined in stub"
-    ),
-    "no_stubs": "could not find typeshed {error.name}",
-    "inconsistent": (
-        '"{error.name}" is {error.stub_type} in stub but'
-        " {error.module_type} at runtime"
-    ),
-}  # type: Final
-
-Error = NamedTuple(
-    "Error",
-    (
-        ("module", str),
-        ("name", str),
-        ("error_type", str),
-        ("line", Optional[int]),
-        ("stub_type", Optional[Type[nodes.Node]]),
-        ("module_type", Optional[str]),
-    ),
-)
-
-ErrorParts = Tuple[
-    List[str], str, Optional[int], Optional[Type[nodes.Node]], Optional[str],
-]
+class Error(str):
+    pass
 
 
 def test_module(
@@ -77,7 +31,6 @@ def test_module(
         mod: stub
         for mod, stub in build_stubs(module_name, options, find_module_cache).items()
         if (mod == module_name or mod.startswith(module_name + "."))
-        and mod not in MODULES_TO_SKIP
     }
 
     for mod, stub in stubs.items():
@@ -86,18 +39,16 @@ def test_module(
 
 
 @singledispatch
-def verify(stub: nodes.Node, runtime: Optional[Any]) -> Iterator[ErrorParts]:
+def verify(stub: nodes.Node, runtime: Optional[Any]) -> Iterator[Error]:
     raise TypeError("unknown mypy node " + str(stub))
 
 
 @verify.register(nodes.MypyFile)
-def verify_mypyfile(
-    stub: nodes.MypyFile, runtime: Optional[Any]
-) -> Iterator[ErrorParts]:
+def verify_mypyfile(stub: nodes.MypyFile, runtime: Optional[Any]) -> Iterator[Error]:
     if runtime is None:
-        yield [], "not_in_runtime", stub.line, type(stub), None
+        yield Error("not_in_runtime")
     elif runtime["type"] != "file":
-        yield [], "inconsistent", stub.line, type(stub), runtime["type"]
+        yield Error("inconsistent")
     else:
         stub_children = defaultdict(
             lambda: None, stub.names
@@ -116,50 +67,40 @@ def verify_mypyfile(
 
         for node, (stub_child, runtime_child) in public_nodes.items():
             stub_child = getattr(stub_child, "node", None)
-            for identifiers, error_type, line, stub_type, module_type in verify(
-                stub_child, runtime_child
-            ):
-                yield ([node] + identifiers, error_type, line, stub_type, module_type)
+            yield from verify(stub_child, runtime_child)
 
 
 @verify.register(nodes.TypeInfo)
-def verify_typeinfo(
-    stub: nodes.TypeInfo, runtime: Optional[Any]
-) -> Iterator[ErrorParts]:
+def verify_typeinfo(stub: nodes.TypeInfo, runtime: Optional[Any]) -> Iterator[Error]:
     if not runtime:
-        yield [], "not_in_runtime", stub.line, type(stub), None
+        yield Error("not_in_runtime")
     elif runtime["type"] != "class":
-        yield [], "inconsistent", stub.line, type(stub), runtime["type"]
+        yield Error("inconsistent")
     else:
         for attr, attr_node in stub.names.items():
             subdump = runtime["attributes"].get(attr, None)
-            for identifiers, error_type, line, stub_type, module_type in verify(
-                attr_node.node, subdump
-            ):
-                yield ([attr] + identifiers, error_type, line, stub_type, module_type)
+            yield from verify(attr_node.node, subdump)
 
 
 @verify.register(nodes.FuncItem)
-def verify_funcitem(
-    stub: nodes.FuncItem, runtime: Optional[Any]
-) -> Iterator[ErrorParts]:
+def verify_funcitem(stub: nodes.FuncItem, runtime: Optional[Any]) -> Iterator[Error]:
     if not runtime:
-        yield [], "not_in_runtime", stub.line, type(stub), None
+        yield Error("not_in_runtime")
     elif "type" not in runtime or runtime["type"] not in ("function", "callable"):
-        yield [], "inconsistent", stub.line, type(stub), runtime["type"]
+        yield Error("inconsistent")
     # TODO check arguments and return value
 
 
 @verify.register(type(None))
-def verify_none(stub: None, runtime: Optional[Any]) -> Iterator[ErrorParts]:
+def verify_none(stub: None, runtime: Optional[Any]) -> Iterator[Error]:
     if runtime is None:
-        yield [], "not_in_stub", None, None, None
+        yield Error("not_in_stub")
     else:
-        yield [], "not_in_stub", runtime["line"], None, runtime["type"]
+        yield Error("not_in_stub")
 
 
 @verify.register(nodes.Var)
-def verify_var(node: nodes.Var, module_node: Optional[Any]) -> Iterator[ErrorParts]:
+def verify_var(node: nodes.Var, module_node: Optional[Any]) -> Iterator[Error]:
     if False:
         yield None
     # Need to check if types are inconsistent.
@@ -171,7 +112,7 @@ def verify_var(node: nodes.Var, module_node: Optional[Any]) -> Iterator[ErrorPar
 @verify.register(nodes.OverloadedFuncDef)
 def verify_overloadedfuncdef(
     node: nodes.OverloadedFuncDef, module_node: Optional[Any]
-) -> Iterator[ErrorParts]:
+) -> Iterator[Error]:
     # Should check types of the union of the overloaded types.
     if False:
         yield None
@@ -180,7 +121,7 @@ def verify_overloadedfuncdef(
 @verify.register(nodes.TypeVarExpr)
 def verify_typevarexpr(
     node: nodes.TypeVarExpr, module_node: Optional[Any]
-) -> Iterator[ErrorParts]:
+) -> Iterator[Error]:
     if False:
         yield None
 
@@ -188,7 +129,7 @@ def verify_typevarexpr(
 @verify.register(nodes.Decorator)
 def verify_decorator(
     node: nodes.Decorator, module_node: Optional[Any]
-) -> Iterator[ErrorParts]:
+) -> Iterator[Error]:
     if False:
         yield None
 
@@ -196,7 +137,7 @@ def verify_decorator(
 @verify.register(nodes.TypeAlias)
 def verify_typealias(
     node: nodes.TypeAlias, module_node: Optional[Any]
-) -> Iterator[ErrorParts]:
+) -> Iterator[Error]:
     if False:
         yield None
 
@@ -205,16 +146,11 @@ def build_stubs(
     module_name: str, options: Options, find_module_cache: FindModuleCache
 ) -> Dict[str, nodes.MypyFile]:
     sources = find_module_cache.find_modules_recursive(module_name)
-    try:
-        res = mypy.build.build(sources=sources, options=options)
-        messages = res.errors
-    except CompileError as error:
-        messages = error.messages
-
-    if messages:
-        for msg in messages:
-            print(msg)
-        sys.exit(1)
+
+    res = mypy.build.build(sources=sources, options=options)
+    if res.errors:
+        raise CompileError
+
     return res.files
 
 
@@ -241,4 +177,4 @@ def main() -> Iterator[Error]:
 
 if __name__ == "__main__":
     for err in main():
-        print(messages[err.error_type].format(error=err))
+        print(err)

From 8deb02a5d32ee85ef7dae0e4094bf12c08f3f877 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 21:18:42 -0800
Subject: [PATCH 11/74] stubtest: add support for missing things

---
 scripts/stubtest.py | 58 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 44 insertions(+), 14 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index dda2dbb0eb41..08bcb1e3fbca 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -7,19 +7,43 @@
 import argparse
 import importlib
 import sys
-from typing import Dict, Any, List, Iterator, NamedTuple, Optional, Mapping, Tuple
-from typing_extensions import Type, Final
+import types
 from collections import defaultdict
 from functools import singledispatch
+from typing import (
+    Any,
+    Dict,
+    Iterator,
+    List,
+    Mapping,
+    NamedTuple,
+    Optional,
+    Tuple,
+    TypeVar,
+    Union,
+)
+
+from typing_extensions import Final, Type
 
 import mypy.build
 import mypy.modulefinder
-from mypy.modulefinder import FindModuleCache
-from mypy.errors import CompileError
 from mypy import nodes
+from mypy.errors import CompileError
+from mypy.modulefinder import FindModuleCache
 from mypy.options import Options
 
 
+class Missing:
+    def __repr__(self) -> str:
+        return "MISSING"
+
+
+MISSING = Missing()
+
+T = TypeVar("T")
+MaybeMissing = Union[T, Missing]
+
+
 class Error(str):
     pass
 
@@ -39,12 +63,14 @@ def test_module(
 
 
 @singledispatch
-def verify(stub: nodes.Node, runtime: Optional[Any]) -> Iterator[Error]:
+def verify(stub: nodes.Node, runtime: MaybeMissing[Any]) -> Iterator[Error]:
     raise TypeError("unknown mypy node " + str(stub))
 
 
 @verify.register(nodes.MypyFile)
-def verify_mypyfile(stub: nodes.MypyFile, runtime: Optional[Any]) -> Iterator[Error]:
+def verify_mypyfile(
+    stub: nodes.MypyFile, runtime: MaybeMissing[types.ModuleType]
+) -> Iterator[Error]:
     if runtime is None:
         yield Error("not_in_runtime")
     elif runtime["type"] != "file":
@@ -71,7 +97,9 @@ def verify_mypyfile(stub: nodes.MypyFile, runtime: Optional[Any]) -> Iterator[Er
 
 
 @verify.register(nodes.TypeInfo)
-def verify_typeinfo(stub: nodes.TypeInfo, runtime: Optional[Any]) -> Iterator[Error]:
+def verify_typeinfo(
+    stub: nodes.TypeInfo, runtime: MaybeMissing[Any]
+) -> Iterator[Error]:
     if not runtime:
         yield Error("not_in_runtime")
     elif runtime["type"] != "class":
@@ -83,7 +111,9 @@ def verify_typeinfo(stub: nodes.TypeInfo, runtime: Optional[Any]) -> Iterator[Er
 
 
 @verify.register(nodes.FuncItem)
-def verify_funcitem(stub: nodes.FuncItem, runtime: Optional[Any]) -> Iterator[Error]:
+def verify_funcitem(
+    stub: nodes.FuncItem, runtime: MaybeMissing[Any]
+) -> Iterator[Error]:
     if not runtime:
         yield Error("not_in_runtime")
     elif "type" not in runtime or runtime["type"] not in ("function", "callable"):
@@ -92,7 +122,7 @@ def verify_funcitem(stub: nodes.FuncItem, runtime: Optional[Any]) -> Iterator[Er
 
 
 @verify.register(type(None))
-def verify_none(stub: None, runtime: Optional[Any]) -> Iterator[Error]:
+def verify_none(stub: None, runtime: MaybeMissing[Any]) -> Iterator[Error]:
     if runtime is None:
         yield Error("not_in_stub")
     else:
@@ -100,7 +130,7 @@ def verify_none(stub: None, runtime: Optional[Any]) -> Iterator[Error]:
 
 
 @verify.register(nodes.Var)
-def verify_var(node: nodes.Var, module_node: Optional[Any]) -> Iterator[Error]:
+def verify_var(node: nodes.Var, module_node: MaybeMissing[Any]) -> Iterator[Error]:
     if False:
         yield None
     # Need to check if types are inconsistent.
@@ -111,7 +141,7 @@ def verify_var(node: nodes.Var, module_node: Optional[Any]) -> Iterator[Error]:
 
 @verify.register(nodes.OverloadedFuncDef)
 def verify_overloadedfuncdef(
-    node: nodes.OverloadedFuncDef, module_node: Optional[Any]
+    node: nodes.OverloadedFuncDef, module_node: MaybeMissing[Any]
 ) -> Iterator[Error]:
     # Should check types of the union of the overloaded types.
     if False:
@@ -120,7 +150,7 @@ def verify_overloadedfuncdef(
 
 @verify.register(nodes.TypeVarExpr)
 def verify_typevarexpr(
-    node: nodes.TypeVarExpr, module_node: Optional[Any]
+    node: nodes.TypeVarExpr, module_node: MaybeMissing[Any]
 ) -> Iterator[Error]:
     if False:
         yield None
@@ -128,7 +158,7 @@ def verify_typevarexpr(
 
 @verify.register(nodes.Decorator)
 def verify_decorator(
-    node: nodes.Decorator, module_node: Optional[Any]
+    node: nodes.Decorator, module_node: MaybeMissing[Any]
 ) -> Iterator[Error]:
     if False:
         yield None
@@ -136,7 +166,7 @@ def verify_decorator(
 
 @verify.register(nodes.TypeAlias)
 def verify_typealias(
-    node: nodes.TypeAlias, module_node: Optional[Any]
+    node: nodes.TypeAlias, module_node: MaybeMissing[Any]
 ) -> Iterator[Error]:
     if False:
         yield None

From 91391bb49392691afb7ab25beb574b47535bd8ec Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 21:47:35 -0800
Subject: [PATCH 12/74] stubtest: implement verify module

---
 scripts/stubtest.py | 46 +++++++++++++++++++++++----------------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 08bcb1e3fbca..f1df9efe13d4 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -71,35 +71,37 @@ def verify(stub: nodes.Node, runtime: MaybeMissing[Any]) -> Iterator[Error]:
 def verify_mypyfile(
     stub: nodes.MypyFile, runtime: MaybeMissing[types.ModuleType]
 ) -> Iterator[Error]:
-    if runtime is None:
+    if isinstance(runtime, Missing):
         yield Error("not_in_runtime")
-    elif runtime["type"] != "file":
-        yield Error("inconsistent")
-    else:
-        stub_children = defaultdict(
-            lambda: None, stub.names
-        )  # type: Mapping[str, Optional[nodes.SymbolTableNode]]
-        runtime_children = defaultdict(lambda: None, runtime["names"])
-
-        # TODO: I would rather not filter public children here.
-        #       For example, what if the checkersurfaces an inconsistency
-        #       in the typing of a private child
-        public_nodes = {
-            name: (stub_children[name], runtime_children[name])
-            for name in set(stub_children) | set(runtime_children)
-            if not name.startswith("_")
-            and (stub_children[name] is None or stub_children[name].module_public)  # type: ignore
-        }
-
-        for node, (stub_child, runtime_child) in public_nodes.items():
-            stub_child = getattr(stub_child, "node", None)
-            yield from verify(stub_child, runtime_child)
+        return
+    if not isinstance(runtime, types.ModuleType):
+        yield Error("type_mismatch")
+        return
+
+    # Check all things in the stub
+    to_check = set(m for m, o in stub.names.items() if o.module_public)
+    # Check all things declared in module's __all__
+    to_check.update(getattr(runtime, "__all__", []))
+    to_check.difference_update(
+        {"__file__", "__doc__", "__name__", "__builtins__", "__package__"}
+    )
+    # We currently don't check things in the module that aren't in the stub, other than things that
+    # are in __all__ to avoid false positives.
+
+    for entry in to_check:
+        yield from verify(
+            getattr(stub.names.get(entry, MISSING), "node", MISSING),
+            getattr(runtime, entry, MISSING),
+        )
 
 
 @verify.register(nodes.TypeInfo)
 def verify_typeinfo(
     stub: nodes.TypeInfo, runtime: MaybeMissing[Any]
 ) -> Iterator[Error]:
+    if isinstance(runtime, Missing):
+        yield Error("not_in_runtime")
+        return
     if not runtime:
         yield Error("not_in_runtime")
     elif runtime["type"] != "class":

From 092961f6c2715ec5e0289f9d25d4e646f9149a19 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 21:49:37 -0800
Subject: [PATCH 13/74] stubtest: add trace for easier debugging

---
 scripts/stubtest.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index f1df9efe13d4..1f7a37ce4b28 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -62,12 +62,23 @@ def test_module(
         yield from verify(stub, runtime)
 
 
+def trace(fn):
+    import functools
+
+    @functools.wraps(fn)
+    def new_fn(*args, **kwargs):
+        return fn(*args, **kwargs)
+
+    return new_fn
+
+
 @singledispatch
 def verify(stub: nodes.Node, runtime: MaybeMissing[Any]) -> Iterator[Error]:
     raise TypeError("unknown mypy node " + str(stub))
 
 
 @verify.register(nodes.MypyFile)
+@trace
 def verify_mypyfile(
     stub: nodes.MypyFile, runtime: MaybeMissing[types.ModuleType]
 ) -> Iterator[Error]:
@@ -96,6 +107,7 @@ def verify_mypyfile(
 
 
 @verify.register(nodes.TypeInfo)
+@trace
 def verify_typeinfo(
     stub: nodes.TypeInfo, runtime: MaybeMissing[Any]
 ) -> Iterator[Error]:
@@ -113,6 +125,7 @@ def verify_typeinfo(
 
 
 @verify.register(nodes.FuncItem)
+@trace
 def verify_funcitem(
     stub: nodes.FuncItem, runtime: MaybeMissing[Any]
 ) -> Iterator[Error]:
@@ -124,6 +137,7 @@ def verify_funcitem(
 
 
 @verify.register(type(None))
+@trace
 def verify_none(stub: None, runtime: MaybeMissing[Any]) -> Iterator[Error]:
     if runtime is None:
         yield Error("not_in_stub")
@@ -132,6 +146,7 @@ def verify_none(stub: None, runtime: MaybeMissing[Any]) -> Iterator[Error]:
 
 
 @verify.register(nodes.Var)
+@trace
 def verify_var(node: nodes.Var, module_node: MaybeMissing[Any]) -> Iterator[Error]:
     if False:
         yield None
@@ -142,6 +157,7 @@ def verify_var(node: nodes.Var, module_node: MaybeMissing[Any]) -> Iterator[Erro
 
 
 @verify.register(nodes.OverloadedFuncDef)
+@trace
 def verify_overloadedfuncdef(
     node: nodes.OverloadedFuncDef, module_node: MaybeMissing[Any]
 ) -> Iterator[Error]:
@@ -151,6 +167,7 @@ def verify_overloadedfuncdef(
 
 
 @verify.register(nodes.TypeVarExpr)
+@trace
 def verify_typevarexpr(
     node: nodes.TypeVarExpr, module_node: MaybeMissing[Any]
 ) -> Iterator[Error]:
@@ -159,6 +176,7 @@ def verify_typevarexpr(
 
 
 @verify.register(nodes.Decorator)
+@trace
 def verify_decorator(
     node: nodes.Decorator, module_node: MaybeMissing[Any]
 ) -> Iterator[Error]:
@@ -167,6 +185,7 @@ def verify_decorator(
 
 
 @verify.register(nodes.TypeAlias)
+@trace
 def verify_typealias(
     node: nodes.TypeAlias, module_node: MaybeMissing[Any]
 ) -> Iterator[Error]:

From 6eae4db7288f7455b9e1d0a806f6fc25e77d385f Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 22:11:49 -0800
Subject: [PATCH 14/74] stubtest: implement verify class

---
 scripts/stubtest.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 1f7a37ce4b28..fb591a20d126 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -109,19 +109,23 @@ def verify_mypyfile(
 @verify.register(nodes.TypeInfo)
 @trace
 def verify_typeinfo(
-    stub: nodes.TypeInfo, runtime: MaybeMissing[Any]
+    stub: nodes.TypeInfo, runtime: MaybeMissing[Type[Any]]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
         yield Error("not_in_runtime")
         return
-    if not runtime:
-        yield Error("not_in_runtime")
-    elif runtime["type"] != "class":
-        yield Error("inconsistent")
-    else:
-        for attr, attr_node in stub.names.items():
-            subdump = runtime["attributes"].get(attr, None)
-            yield from verify(attr_node.node, subdump)
+    if not isinstance(runtime, type):
+        yield Error("type_mismatch")
+        return
+
+    to_check = set(stub.names)
+    to_check.update(m for m in vars(runtime) if not m.startswith("_"))
+
+    for entry in to_check:
+        yield from verify(
+            getattr(stub.names.get(entry, MISSING), "node", MISSING),
+            getattr(runtime, entry, MISSING),
+        )
 
 
 @verify.register(nodes.FuncItem)

From fd9b1d162ae744065fb71da3288e6d961ce10f1a Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 22:24:06 -0800
Subject: [PATCH 15/74] stubtest: implement verify missing

---
 scripts/stubtest.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index fb591a20d126..ba556de1cc52 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -74,7 +74,7 @@ def new_fn(*args, **kwargs):
 
 @singledispatch
 def verify(stub: nodes.Node, runtime: MaybeMissing[Any]) -> Iterator[Error]:
-    raise TypeError("unknown mypy node " + str(stub))
+    print("unknown mypy node " + str(stub))
 
 
 @verify.register(nodes.MypyFile)
@@ -140,13 +140,12 @@ def verify_funcitem(
     # TODO check arguments and return value
 
 
-@verify.register(type(None))
+@verify.register(Missing)
 @trace
-def verify_none(stub: None, runtime: MaybeMissing[Any]) -> Iterator[Error]:
-    if runtime is None:
-        yield Error("not_in_stub")
-    else:
-        yield Error("not_in_stub")
+def verify_none(stub: Missing, runtime: MaybeMissing[Any]) -> Iterator[Error]:
+    yield Error(f"not_in_stub: {runtime}")
+    if isinstance(runtime, Missing):
+        raise RuntimeError
 
 
 @verify.register(nodes.Var)

From c9ef96a891fbe6c370d8bd2aecd60714a14b48c4 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 23:34:14 -0800
Subject: [PATCH 16/74] stubtest: implement verify function

---
 scripts/stubtest.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index ba556de1cc52..e1022899f9b3 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -6,6 +6,7 @@
 
 import argparse
 import importlib
+import inspect
 import sys
 import types
 from collections import defaultdict
@@ -131,13 +132,25 @@ def verify_typeinfo(
 @verify.register(nodes.FuncItem)
 @trace
 def verify_funcitem(
-    stub: nodes.FuncItem, runtime: MaybeMissing[Any]
+    stub: nodes.FuncItem, runtime: MaybeMissing[types.FunctionType]
 ) -> Iterator[Error]:
-    if not runtime:
+    if isinstance(runtime, Missing):
         yield Error("not_in_runtime")
-    elif "type" not in runtime or runtime["type"] not in ("function", "callable"):
-        yield Error("inconsistent")
-    # TODO check arguments and return value
+        return
+    if not isinstance(runtime, (types.FunctionType, types.BuiltinFunctionType)):
+        yield Error(f"type_mismatch: {runtime}")
+        return
+
+    # TODO: make this better
+    try:
+        runtime_params = set(inspect.signature(runtime).parameters)
+        stub_params = set(arg.variable.name for arg in stub.arguments)
+        todo = runtime_params.symmetric_difference(stub_params)
+        if todo:
+            yield Error(f"arg_mismatch: {todo} at {runtime}")
+    except ValueError:
+        # inspect.signature throws sometimes
+        pass
 
 
 @verify.register(Missing)

From fe754851fbff85bf86aa7501f4475a2cafd4bc92 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 15 Jan 2020 23:57:01 -0800
Subject: [PATCH 17/74] stubtest: implement verify var

---
 scripts/stubtest.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index e1022899f9b3..06daf5e487ac 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -28,6 +28,7 @@
 
 import mypy.build
 import mypy.modulefinder
+import mypy.types
 from mypy import nodes
 from mypy.errors import CompileError
 from mypy.modulefinder import FindModuleCache
@@ -163,19 +164,20 @@ def verify_none(stub: Missing, runtime: MaybeMissing[Any]) -> Iterator[Error]:
 
 @verify.register(nodes.Var)
 @trace
-def verify_var(node: nodes.Var, module_node: MaybeMissing[Any]) -> Iterator[Error]:
-    if False:
-        yield None
-    # Need to check if types are inconsistent.
-    # if 'type' not in dump or dump['type'] != node.node.type:
-    #    import ipdb; ipdb.set_trace()
-    #    yield name, 'inconsistent', node.node.line, shed_type, module_type
+def verify_var(stub: nodes.Var, runtime: MaybeMissing[Any]) -> Iterator[Error]:
+    if isinstance(runtime, Missing):
+        yield Error("not_in_runtime")
+        return
+    # TODO: Make this better
+    if isinstance(stub, mypy.types.Instance):
+        if stub.type.type.name != runtime.__name__:
+            yield Error(f"var_mismatch: {runtime}")
 
 
 @verify.register(nodes.OverloadedFuncDef)
 @trace
 def verify_overloadedfuncdef(
-    node: nodes.OverloadedFuncDef, module_node: MaybeMissing[Any]
+    stub: nodes.OverloadedFuncDef, runtime: MaybeMissing[Any]
 ) -> Iterator[Error]:
     # Should check types of the union of the overloaded types.
     if False:
@@ -185,7 +187,7 @@ def verify_overloadedfuncdef(
 @verify.register(nodes.TypeVarExpr)
 @trace
 def verify_typevarexpr(
-    node: nodes.TypeVarExpr, module_node: MaybeMissing[Any]
+    stub: nodes.TypeVarExpr, runtime: MaybeMissing[Any]
 ) -> Iterator[Error]:
     if False:
         yield None
@@ -194,7 +196,7 @@ def verify_typevarexpr(
 @verify.register(nodes.Decorator)
 @trace
 def verify_decorator(
-    node: nodes.Decorator, module_node: MaybeMissing[Any]
+    stub: nodes.Decorator, runtime: MaybeMissing[Any]
 ) -> Iterator[Error]:
     if False:
         yield None
@@ -203,7 +205,7 @@ def verify_decorator(
 @verify.register(nodes.TypeAlias)
 @trace
 def verify_typealias(
-    node: nodes.TypeAlias, module_node: MaybeMissing[Any]
+    stub: nodes.TypeAlias, runtime: MaybeMissing[Any]
 ) -> Iterator[Error]:
     if False:
         yield None

From 53f768e75674140420911a55cb2a8052fcdc3b47 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 00:05:45 -0800
Subject: [PATCH 18/74] stubtest: logging improvements

---
 scripts/stubtest.py | 90 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 77 insertions(+), 13 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 06daf5e487ac..767964f73ad6 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -13,6 +13,7 @@
 from functools import singledispatch
 from typing import (
     Any,
+    Callable,
     Dict,
     Iterator,
     List,
@@ -33,6 +34,7 @@
 from mypy.errors import CompileError
 from mypy.modulefinder import FindModuleCache
 from mypy.options import Options
+from mypy.util import FancyFormatter
 
 
 class Missing:
@@ -46,8 +48,68 @@ def __repr__(self) -> str:
 MaybeMissing = Union[T, Missing]
 
 
-class Error(str):
-    pass
+class Error:
+    def __init__(
+        self,
+        message: str,
+        stub_object: MaybeMissing[nodes.Node],
+        runtime_object: MaybeMissing[Any],
+        stub_printer: Optional[Callable[[nodes.Node], str]] = None,
+        runtime_printer: Optional[Callable[[Any], str]] = None,
+    ) -> None:
+        self.message = message
+        self.stub_object = stub_object
+        self.runtime_object = runtime_object
+        if stub_printer is None:
+            stub_printer = lambda stub: str(stub.type)
+        self.stub_printer = lambda s: s if isinstance(s, Missing) else stub_printer(s)
+        if runtime_printer is None:
+            runtime_printer = lambda runtime: str(runtime)
+        self.runtime_printer = (
+            lambda s: s if isinstance(s, Missing) else runtime_printer(s)
+        )
+
+    def __str__(self) -> str:
+        stub_line = None
+        stub_file = None
+        if not isinstance(self.stub_object, Missing):
+            stub_line = self.stub_object.line
+        # TODO: Find a way of getting the stub file
+
+        stub_loc_str = ""
+        if stub_line:
+            stub_loc_str += f" at line {stub_line}"
+        if stub_file:
+            stub_loc_str += f" in file {stub_file}"
+
+        runtime_line = None
+        runtime_file = None
+        if not isinstance(self.runtime_object, Missing):
+            try:
+                runtime_line = inspect.getsourcelines(self.runtime_object)[1]
+            except (OSError, TypeError):
+                pass
+            try:
+                runtime_file = inspect.getsourcefile(self.runtime_object)
+            except TypeError:
+                pass
+
+        runtime_loc_str = ""
+        if runtime_line:
+            runtime_loc_str += f" at line {runtime_line}"
+        if runtime_file:
+            runtime_loc_str += f" in file {runtime_file}"
+
+        return "".join(
+            [
+                formatter.style("error:", "red", bold=True),
+                f" {self.message}\n",
+                f"Stub{stub_loc_str}:\n",
+                f"{self.stub_printer(self.stub_object)}\n",
+                f"Runtime{runtime_loc_str}:\n",
+                f"{self.runtime_printer(self.runtime_object)}\n",
+            ]
+        )
 
 
 def test_module(
@@ -76,7 +138,7 @@ def new_fn(*args, **kwargs):
 
 @singledispatch
 def verify(stub: nodes.Node, runtime: MaybeMissing[Any]) -> Iterator[Error]:
-    print("unknown mypy node " + str(stub))
+    yield Error("unknown mypy node", stub, runtime)
 
 
 @verify.register(nodes.MypyFile)
@@ -85,10 +147,10 @@ def verify_mypyfile(
     stub: nodes.MypyFile, runtime: MaybeMissing[types.ModuleType]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
-        yield Error("not_in_runtime")
+        yield Error("not_in_runtime", stub, runtime)
         return
     if not isinstance(runtime, types.ModuleType):
-        yield Error("type_mismatch")
+        yield Error("type_mismatch", stub, runtime)
         return
 
     # Check all things in the stub
@@ -114,10 +176,10 @@ def verify_typeinfo(
     stub: nodes.TypeInfo, runtime: MaybeMissing[Type[Any]]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
-        yield Error("not_in_runtime")
+        yield Error("not_in_runtime", stub, runtime)
         return
     if not isinstance(runtime, type):
-        yield Error("type_mismatch")
+        yield Error("type_mismatch", stub, runtime)
         return
 
     to_check = set(stub.names)
@@ -136,10 +198,10 @@ def verify_funcitem(
     stub: nodes.FuncItem, runtime: MaybeMissing[types.FunctionType]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
-        yield Error("not_in_runtime")
+        yield Error("not_in_runtime", stub, runtime)
         return
     if not isinstance(runtime, (types.FunctionType, types.BuiltinFunctionType)):
-        yield Error(f"type_mismatch: {runtime}")
+        yield Error("type_mismatch", stub, runtime)
         return
 
     # TODO: make this better
@@ -148,7 +210,7 @@ def verify_funcitem(
         stub_params = set(arg.variable.name for arg in stub.arguments)
         todo = runtime_params.symmetric_difference(stub_params)
         if todo:
-            yield Error(f"arg_mismatch: {todo} at {runtime}")
+            yield Error(f"arg_mismatch: {todo}", stub, runtime)
     except ValueError:
         # inspect.signature throws sometimes
         pass
@@ -157,7 +219,7 @@ def verify_funcitem(
 @verify.register(Missing)
 @trace
 def verify_none(stub: Missing, runtime: MaybeMissing[Any]) -> Iterator[Error]:
-    yield Error(f"not_in_stub: {runtime}")
+    yield Error("not_in_stub", stub, runtime)
     if isinstance(runtime, Missing):
         raise RuntimeError
 
@@ -166,12 +228,13 @@ def verify_none(stub: Missing, runtime: MaybeMissing[Any]) -> Iterator[Error]:
 @trace
 def verify_var(stub: nodes.Var, runtime: MaybeMissing[Any]) -> Iterator[Error]:
     if isinstance(runtime, Missing):
-        yield Error("not_in_runtime")
+        # Don't yield an error here, because we often can't find instance variables
+        # yield Error("not_in_runtime", stub, runtime)
         return
     # TODO: Make this better
     if isinstance(stub, mypy.types.Instance):
         if stub.type.type.name != runtime.__name__:
-            yield Error(f"var_mismatch: {runtime}")
+            yield Error(f"var_mismatch", stub, runtime)
 
 
 @verify.register(nodes.OverloadedFuncDef)
@@ -245,5 +308,6 @@ def main() -> Iterator[Error]:
 
 
 if __name__ == "__main__":
+    formatter = FancyFormatter(sys.stdout, sys.stderr, False)
     for err in main():
         print(err)

From ac63aeed954600c19782cf2954c190a75acb69b5 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 08:24:17 -0800
Subject: [PATCH 19/74] stubtest: improve verify function

---
 scripts/stubtest.py | 73 ++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 66 insertions(+), 7 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 767964f73ad6..32d3c83442da 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -204,16 +204,75 @@ def verify_funcitem(
         yield Error("type_mismatch", stub, runtime)
         return
 
-    # TODO: make this better
     try:
-        runtime_params = set(inspect.signature(runtime).parameters)
-        stub_params = set(arg.variable.name for arg in stub.arguments)
-        todo = runtime_params.symmetric_difference(stub_params)
-        if todo:
-            yield Error(f"arg_mismatch: {todo}", stub, runtime)
+        signature = inspect.signature(runtime)
     except ValueError:
         # inspect.signature throws sometimes
-        pass
+        return
+
+    i, j = 0, 0
+    stub_args = stub.arguments
+    runtime_args = list(signature.parameters.values())
+    while i < len(stub_args) or j < len(runtime_args):
+        if i >= len(stub_args):
+            # Ignore errors if the stub doesn't take **kwargs, since the stub might just have
+            # listed out the keyword parameters this function takes
+            if runtime_args[j].kind != inspect.Parameter.VAR_KEYWORD:
+                yield Error(
+                    f"arg_mismatch: {stub.name}, stub missing {runtime_args[j].name}",
+                    stub,
+                    runtime,
+                    runtime_printer=lambda s: str(inspect.signature(s)),
+                )
+            j += 1
+            continue
+        if j >= len(runtime_args):
+            yield Error(
+                f"arg_mismatch: {stub.name}, runtime missing {stub_args[i].variable.name}",
+                stub,
+                runtime,
+                runtime_printer=lambda s: str(inspect.signature(s)),
+            )
+            i += 1
+            continue
+
+        # TODO: maybe don't check by name for positional-only args, dunder methods
+        # TODO: stricter checking of positional-only, keyword-only
+        # TODO: check type compatibility of default args
+
+        stub_arg, runtime_arg = stub_args[i], runtime_args[j]
+        if (stub_arg.kind == mypy.nodes.ARG_STAR) and (
+            runtime_arg.kind != inspect.Parameter.VAR_POSITIONAL
+        ):
+            j += 1
+            continue
+        if (stub_arg.kind != mypy.nodes.ARG_STAR) and (
+            runtime_arg.kind == inspect.Parameter.VAR_POSITIONAL
+        ):
+            i += 1
+            continue
+
+        if (stub_arg.kind == mypy.nodes.ARG_STAR2) and (
+            runtime_arg.kind != inspect.Parameter.VAR_KEYWORD
+        ):
+            j += 1
+            continue
+        if (stub_arg.kind != mypy.nodes.ARG_STAR2) and (
+            runtime_arg.kind == inspect.Parameter.VAR_KEYWORD
+        ):
+            i += 1
+            continue
+
+        if stub_arg.variable.name != runtime_arg.name:
+            # TODO: _asdict takes _self and self
+            yield Error(
+                f"arg_mismatch: {stub.name}, {stub_arg.variable.name} != {runtime_arg.name}",
+                stub,
+                runtime,
+                runtime_printer=lambda s: str(inspect.signature(s)),
+            )
+        i += 1
+        j += 1
 
 
 @verify.register(Missing)

From eedb3f2db5dadfc2e79ce163b0ae99373c980b49 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 08:47:21 -0800
Subject: [PATCH 20/74] stubtest: implement verify overload

---
 scripts/stubtest.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 32d3c83442da..3dc06336a816 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -239,6 +239,7 @@ def verify_funcitem(
         # TODO: maybe don't check by name for positional-only args, dunder methods
         # TODO: stricter checking of positional-only, keyword-only
         # TODO: check type compatibility of default args
+        # TODO: overloads are sometimes pretty deceitful, so handle that better
 
         stub_arg, runtime_arg = stub_args[i], runtime_args[j]
         if (stub_arg.kind == mypy.nodes.ARG_STAR) and (
@@ -301,9 +302,8 @@ def verify_var(stub: nodes.Var, runtime: MaybeMissing[Any]) -> Iterator[Error]:
 def verify_overloadedfuncdef(
     stub: nodes.OverloadedFuncDef, runtime: MaybeMissing[Any]
 ) -> Iterator[Error]:
-    # Should check types of the union of the overloaded types.
-    if False:
-        yield None
+    for func in stub.items:
+        yield from verify(func, runtime)
 
 
 @verify.register(nodes.TypeVarExpr)
@@ -320,8 +320,12 @@ def verify_typevarexpr(
 def verify_decorator(
     stub: nodes.Decorator, runtime: MaybeMissing[Any]
 ) -> Iterator[Error]:
-    if False:
-        yield None
+    if (
+        len(stub.decorators) == 1
+        and isinstance(stub.decorators[0], nodes.NameExpr)
+        and stub.decorators[0].fullname == "typing.overload"
+    ):
+        yield from verify(stub.func, runtime)
 
 
 @verify.register(nodes.TypeAlias)

From be5c715b529e0e10f0e5cfcd2efb005fb9f20e38 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 10:52:32 -0800
Subject: [PATCH 21/74] stubtest: more improvements to logging

---
 scripts/stubtest.py | 109 ++++++++++++++++++++++++++++----------------
 1 file changed, 70 insertions(+), 39 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 3dc06336a816..724f63a844ec 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -47,21 +47,30 @@ def __repr__(self) -> str:
 T = TypeVar("T")
 MaybeMissing = Union[T, Missing]
 
+_formatter = FancyFormatter(sys.stdout, sys.stderr, False)
+
+
+def _style(message: str, **kwargs: Any) -> str:
+    kwargs.setdefault("color", "none")
+    return _formatter.style(message, **kwargs)
+
 
 class Error:
     def __init__(
         self,
+        object_path: List[str],
         message: str,
         stub_object: MaybeMissing[nodes.Node],
         runtime_object: MaybeMissing[Any],
         stub_printer: Optional[Callable[[nodes.Node], str]] = None,
         runtime_printer: Optional[Callable[[Any], str]] = None,
     ) -> None:
+        self.object_desc = ".".join(object_path)
         self.message = message
         self.stub_object = stub_object
         self.runtime_object = runtime_object
         if stub_printer is None:
-            stub_printer = lambda stub: str(stub.type)
+            stub_printer = lambda stub: str(getattr(stub, "type", stub))
         self.stub_printer = lambda s: s if isinstance(s, Missing) else stub_printer(s)
         if runtime_printer is None:
             runtime_printer = lambda runtime: str(runtime)
@@ -100,16 +109,22 @@ def __str__(self) -> str:
         if runtime_file:
             runtime_loc_str += f" in file {runtime_file}"
 
-        return "".join(
-            [
-                formatter.style("error:", "red", bold=True),
-                f" {self.message}\n",
-                f"Stub{stub_loc_str}:\n",
-                f"{self.stub_printer(self.stub_object)}\n",
-                f"Runtime{runtime_loc_str}:\n",
-                f"{self.runtime_printer(self.runtime_object)}\n",
-            ]
-        )
+        output = [
+            _style("error: ", color="red", bold=True),
+            _style(self.object_desc, bold=True),
+            f" {self.message}\n",
+            "Stub:",
+            _style(stub_loc_str, dim=True),
+            "\n",
+            _style(f"{self.stub_printer(self.stub_object)}\n", color="blue", dim=True),
+            "Runtime:",
+            _style(runtime_loc_str, dim=True),
+            "\n",
+            _style(
+                f"{self.runtime_printer(self.runtime_object)}\n", color="blue", dim=True
+            ),
+        ]
+        return "".join(output)
 
 
 def test_module(
@@ -123,7 +138,7 @@ def test_module(
 
     for mod, stub in stubs.items():
         runtime = importlib.import_module(mod)
-        yield from verify(stub, runtime)
+        yield from verify(stub, runtime, [mod])
 
 
 def trace(fn):
@@ -137,20 +152,24 @@ def new_fn(*args, **kwargs):
 
 
 @singledispatch
-def verify(stub: nodes.Node, runtime: MaybeMissing[Any]) -> Iterator[Error]:
-    yield Error("unknown mypy node", stub, runtime)
+def verify(
+    stub: nodes.Node, runtime: MaybeMissing[Any], object_path: List[str]
+) -> Iterator[Error]:
+    yield Error(object_path, "is an unknown mypy node", stub, runtime)
 
 
 @verify.register(nodes.MypyFile)
 @trace
 def verify_mypyfile(
-    stub: nodes.MypyFile, runtime: MaybeMissing[types.ModuleType]
+    stub: nodes.MypyFile,
+    runtime: MaybeMissing[types.ModuleType],
+    object_path: List[str],
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
-        yield Error("not_in_runtime", stub, runtime)
+        yield Error(object_path, "is not present at runtime", stub, runtime)
         return
     if not isinstance(runtime, types.ModuleType):
-        yield Error("type_mismatch", stub, runtime)
+        yield Error(object_path, "is not a module", stub, runtime)
         return
 
     # Check all things in the stub
@@ -167,19 +186,20 @@ def verify_mypyfile(
         yield from verify(
             getattr(stub.names.get(entry, MISSING), "node", MISSING),
             getattr(runtime, entry, MISSING),
+            object_path + [entry],
         )
 
 
 @verify.register(nodes.TypeInfo)
 @trace
 def verify_typeinfo(
-    stub: nodes.TypeInfo, runtime: MaybeMissing[Type[Any]]
+    stub: nodes.TypeInfo, runtime: MaybeMissing[Type[Any]], object_path: List[str]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
-        yield Error("not_in_runtime", stub, runtime)
+        yield Error(object_path, "is not present at runtime", stub, runtime)
         return
     if not isinstance(runtime, type):
-        yield Error("type_mismatch", stub, runtime)
+        yield Error(object_path, "is not a type", stub, runtime)
         return
 
     to_check = set(stub.names)
@@ -189,19 +209,22 @@ def verify_typeinfo(
         yield from verify(
             getattr(stub.names.get(entry, MISSING), "node", MISSING),
             getattr(runtime, entry, MISSING),
+            object_path + [entry],
         )
 
 
 @verify.register(nodes.FuncItem)
 @trace
 def verify_funcitem(
-    stub: nodes.FuncItem, runtime: MaybeMissing[types.FunctionType]
+    stub: nodes.FuncItem,
+    runtime: MaybeMissing[types.FunctionType],
+    object_path: List[str],
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
-        yield Error("not_in_runtime", stub, runtime)
+        yield Error(object_path, "is not present at runtime", stub, runtime)
         return
     if not isinstance(runtime, (types.FunctionType, types.BuiltinFunctionType)):
-        yield Error("type_mismatch", stub, runtime)
+        yield Error(object_path, "is not a function", stub, runtime)
         return
 
     try:
@@ -219,7 +242,8 @@ def verify_funcitem(
             # listed out the keyword parameters this function takes
             if runtime_args[j].kind != inspect.Parameter.VAR_KEYWORD:
                 yield Error(
-                    f"arg_mismatch: {stub.name}, stub missing {runtime_args[j].name}",
+                    object_path,
+                    f'is inconsistent, stub does not have argument "{runtime_args[j].name}"',
                     stub,
                     runtime,
                     runtime_printer=lambda s: str(inspect.signature(s)),
@@ -228,7 +252,8 @@ def verify_funcitem(
             continue
         if j >= len(runtime_args):
             yield Error(
-                f"arg_mismatch: {stub.name}, runtime missing {stub_args[i].variable.name}",
+                object_path,
+                f"is inconsistent, runtime does not have argument {stub_args[i].variable.name}",
                 stub,
                 runtime,
                 runtime_printer=lambda s: str(inspect.signature(s)),
@@ -267,7 +292,9 @@ def verify_funcitem(
         if stub_arg.variable.name != runtime_arg.name:
             # TODO: _asdict takes _self and self
             yield Error(
-                f"arg_mismatch: {stub.name}, {stub_arg.variable.name} != {runtime_arg.name}",
+                object_path,
+                f'is inconsistent, stub argument "{stub_arg.variable.name}" differs from '
+                f'runtime argument "{runtime_arg.name}"',
                 stub,
                 runtime,
                 runtime_printer=lambda s: str(inspect.signature(s)),
@@ -278,38 +305,43 @@ def verify_funcitem(
 
 @verify.register(Missing)
 @trace
-def verify_none(stub: Missing, runtime: MaybeMissing[Any]) -> Iterator[Error]:
-    yield Error("not_in_stub", stub, runtime)
+def verify_none(
+    stub: Missing, runtime: MaybeMissing[Any], object_path: List[str]
+) -> Iterator[Error]:
+    yield Error(object_path, "is not present in stub", stub, runtime)
     if isinstance(runtime, Missing):
         raise RuntimeError
 
 
 @verify.register(nodes.Var)
 @trace
-def verify_var(stub: nodes.Var, runtime: MaybeMissing[Any]) -> Iterator[Error]:
+def verify_var(
+    stub: nodes.Var, runtime: MaybeMissing[Any], object_path: List[str]
+) -> Iterator[Error]:
     if isinstance(runtime, Missing):
-        # Don't yield an error here, because we often can't find instance variables
-        # yield Error("not_in_runtime", stub, runtime)
+        # Don't always yield an error here, because we often can't find instance variables
+        if len(object_path) <= 1:
+            yield Error(object_path, "is not present at runtime", stub, runtime)
         return
     # TODO: Make this better
     if isinstance(stub, mypy.types.Instance):
         if stub.type.type.name != runtime.__name__:
-            yield Error(f"var_mismatch", stub, runtime)
+            yield Error(object_path, "var_mismatch", stub, runtime)
 
 
 @verify.register(nodes.OverloadedFuncDef)
 @trace
 def verify_overloadedfuncdef(
-    stub: nodes.OverloadedFuncDef, runtime: MaybeMissing[Any]
+    stub: nodes.OverloadedFuncDef, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
     for func in stub.items:
-        yield from verify(func, runtime)
+        yield from verify(func, runtime, object_path)
 
 
 @verify.register(nodes.TypeVarExpr)
 @trace
 def verify_typevarexpr(
-    stub: nodes.TypeVarExpr, runtime: MaybeMissing[Any]
+    stub: nodes.TypeVarExpr, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
     if False:
         yield None
@@ -318,20 +350,20 @@ def verify_typevarexpr(
 @verify.register(nodes.Decorator)
 @trace
 def verify_decorator(
-    stub: nodes.Decorator, runtime: MaybeMissing[Any]
+    stub: nodes.Decorator, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
     if (
         len(stub.decorators) == 1
         and isinstance(stub.decorators[0], nodes.NameExpr)
         and stub.decorators[0].fullname == "typing.overload"
     ):
-        yield from verify(stub.func, runtime)
+        yield from verify(stub.func, runtime, object_path)
 
 
 @verify.register(nodes.TypeAlias)
 @trace
 def verify_typealias(
-    stub: nodes.TypeAlias, runtime: MaybeMissing[Any]
+    stub: nodes.TypeAlias, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
     if False:
         yield None
@@ -371,6 +403,5 @@ def main() -> Iterator[Error]:
 
 
 if __name__ == "__main__":
-    formatter = FancyFormatter(sys.stdout, sys.stderr, False)
     for err in main():
         print(err)

From d43cefb3764bf45a3c384cf1eba5f8de02a7f0ba Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 10:55:57 -0800
Subject: [PATCH 22/74] stubtest: add --ignore-missing-stub option

---
 scripts/stubtest.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 724f63a844ec..57a9a3e392ec 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -78,6 +78,9 @@ def __init__(
             lambda s: s if isinstance(s, Missing) else runtime_printer(s)
         )
 
+    def is_missing_stub(self) -> bool:
+        return isinstance(self.stub_object, Missing)
+
     def __str__(self) -> str:
         stub_line = None
         stub_file = None
@@ -384,6 +387,11 @@ def build_stubs(
 def main() -> Iterator[Error]:
     parser = argparse.ArgumentParser()
     parser.add_argument("modules", nargs="+", help="Modules to test")
+    parser.add_argument(
+        "--ignore-missing-stub",
+        action="store_true",
+        help="Ignore errors for stub missing things that are present at runtime",
+    )
     parser.add_argument(
         "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
     )
@@ -399,7 +407,8 @@ def main() -> Iterator[Error]:
 
     for module in args.modules:
         for error in test_module(module, options, find_module_cache):
-            yield error
+            if not args.ignore_missing_stub or not error.is_missing_stub():
+                yield error
 
 
 if __name__ == "__main__":

From be72175b49d6074474341dc8fa99fe7b04a79f10 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 11:19:25 -0800
Subject: [PATCH 23/74] stubtest: [minor] make order more deterministic

---
 scripts/stubtest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 57a9a3e392ec..c62d26383990 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -185,7 +185,7 @@ def verify_mypyfile(
     # We currently don't check things in the module that aren't in the stub, other than things that
     # are in __all__ to avoid false positives.
 
-    for entry in to_check:
+    for entry in sorted(to_check):
         yield from verify(
             getattr(stub.names.get(entry, MISSING), "node", MISSING),
             getattr(runtime, entry, MISSING),
@@ -208,7 +208,7 @@ def verify_typeinfo(
     to_check = set(stub.names)
     to_check.update(m for m in vars(runtime) if not m.startswith("_"))
 
-    for entry in to_check:
+    for entry in sorted(to_check):
         yield from verify(
             getattr(stub.names.get(entry, MISSING), "node", MISSING),
             getattr(runtime, entry, MISSING),

From 7876ca23bcb90046a342a4c8f1990e7501f5cfc4 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 11:23:45 -0800
Subject: [PATCH 24/74] stubtest: [minor] descend through stubs less hackily

---
 scripts/stubtest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index c62d26383990..2167ea866b01 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -187,7 +187,7 @@ def verify_mypyfile(
 
     for entry in sorted(to_check):
         yield from verify(
-            getattr(stub.names.get(entry, MISSING), "node", MISSING),
+            stub.names[entry].node if entry in stub.names else MISSING,
             getattr(runtime, entry, MISSING),
             object_path + [entry],
         )
@@ -210,7 +210,7 @@ def verify_typeinfo(
 
     for entry in sorted(to_check):
         yield from verify(
-            getattr(stub.names.get(entry, MISSING), "node", MISSING),
+            stub.names[entry].node if entry in stub.names else MISSING,
             getattr(runtime, entry, MISSING),
             object_path + [entry],
         )

From 8a41b6df216906bc9a05bd40286f23e822c7b9e8 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 16:37:35 -0800
Subject: [PATCH 25/74] stubtest: [minor] clean up imports

---
 scripts/stubtest.py | 23 +++++------------------
 1 file changed, 5 insertions(+), 18 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 2167ea866b01..8d2c39f89c9e 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -1,7 +1,7 @@
 """Tests for stubs.
 
-Verify that various things in stubs are consistent with how things behave
-at runtime.
+Verify that various things in stubs are consistent with how things behave at runtime.
+
 """
 
 import argparse
@@ -9,23 +9,10 @@
 import inspect
 import sys
 import types
-from collections import defaultdict
 from functools import singledispatch
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Iterator,
-    List,
-    Mapping,
-    NamedTuple,
-    Optional,
-    Tuple,
-    TypeVar,
-    Union,
-)
-
-from typing_extensions import Final, Type
+from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, Union
+
+from typing_extensions import Type
 
 import mypy.build
 import mypy.modulefinder

From f761e8a95270f28cc22feffb70a4d09a4a114dc2 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 17:03:24 -0800
Subject: [PATCH 26/74] stubtest: [minor] remove debugging decorator

---
 scripts/stubtest.py | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 8d2c39f89c9e..eff221b61691 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -131,16 +131,6 @@ def test_module(
         yield from verify(stub, runtime, [mod])
 
 
-def trace(fn):
-    import functools
-
-    @functools.wraps(fn)
-    def new_fn(*args, **kwargs):
-        return fn(*args, **kwargs)
-
-    return new_fn
-
-
 @singledispatch
 def verify(
     stub: nodes.Node, runtime: MaybeMissing[Any], object_path: List[str]
@@ -149,7 +139,6 @@ def verify(
 
 
 @verify.register(nodes.MypyFile)
-@trace
 def verify_mypyfile(
     stub: nodes.MypyFile,
     runtime: MaybeMissing[types.ModuleType],
@@ -181,7 +170,6 @@ def verify_mypyfile(
 
 
 @verify.register(nodes.TypeInfo)
-@trace
 def verify_typeinfo(
     stub: nodes.TypeInfo, runtime: MaybeMissing[Type[Any]], object_path: List[str]
 ) -> Iterator[Error]:
@@ -204,7 +192,6 @@ def verify_typeinfo(
 
 
 @verify.register(nodes.FuncItem)
-@trace
 def verify_funcitem(
     stub: nodes.FuncItem,
     runtime: MaybeMissing[types.FunctionType],
@@ -294,7 +281,6 @@ def verify_funcitem(
 
 
 @verify.register(Missing)
-@trace
 def verify_none(
     stub: Missing, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
@@ -304,7 +290,6 @@ def verify_none(
 
 
 @verify.register(nodes.Var)
-@trace
 def verify_var(
     stub: nodes.Var, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
@@ -320,7 +305,6 @@ def verify_var(
 
 
 @verify.register(nodes.OverloadedFuncDef)
-@trace
 def verify_overloadedfuncdef(
     stub: nodes.OverloadedFuncDef, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
@@ -329,7 +313,6 @@ def verify_overloadedfuncdef(
 
 
 @verify.register(nodes.TypeVarExpr)
-@trace
 def verify_typevarexpr(
     stub: nodes.TypeVarExpr, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
@@ -338,7 +321,6 @@ def verify_typevarexpr(
 
 
 @verify.register(nodes.Decorator)
-@trace
 def verify_decorator(
     stub: nodes.Decorator, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
@@ -351,7 +333,6 @@ def verify_decorator(
 
 
 @verify.register(nodes.TypeAlias)
-@trace
 def verify_typealias(
     stub: nodes.TypeAlias, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:

From 7b5ecfe02f7a2833888f688ac569e5878df33ca5 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 17:46:37 -0800
Subject: [PATCH 27/74] stubtest: small improvements for functions

---
 scripts/stubtest.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index eff221b61691..471c28395984 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -200,7 +200,9 @@ def verify_funcitem(
     if isinstance(runtime, Missing):
         yield Error(object_path, "is not present at runtime", stub, runtime)
         return
-    if not isinstance(runtime, (types.FunctionType, types.BuiltinFunctionType)):
+    if not isinstance(
+        runtime, (types.FunctionType, types.BuiltinFunctionType)
+    ) and not inspect.ismethoddescriptor(runtime):
         yield Error(object_path, "is not a function", stub, runtime)
         return
 
@@ -210,20 +212,23 @@ def verify_funcitem(
         # inspect.signature throws sometimes
         return
 
+    def runtime_printer(s: Any) -> str:
+        return "def " + str(inspect.signature(s))
+
     i, j = 0, 0
     stub_args = stub.arguments
     runtime_args = list(signature.parameters.values())
     while i < len(stub_args) or j < len(runtime_args):
         if i >= len(stub_args):
-            # Ignore errors if the stub doesn't take **kwargs, since the stub might just have
-            # listed out the keyword parameters this function takes
+            # Ignore the error if the stub doesn't take **kwargs, for cases where the stub
+            # just listed out the keyword parameters the function takes
             if runtime_args[j].kind != inspect.Parameter.VAR_KEYWORD:
                 yield Error(
                     object_path,
                     f'is inconsistent, stub does not have argument "{runtime_args[j].name}"',
                     stub,
                     runtime,
-                    runtime_printer=lambda s: str(inspect.signature(s)),
+                    runtime_printer=runtime_printer,
                 )
             j += 1
             continue
@@ -233,16 +238,17 @@ def verify_funcitem(
                 f"is inconsistent, runtime does not have argument {stub_args[i].variable.name}",
                 stub,
                 runtime,
-                runtime_printer=lambda s: str(inspect.signature(s)),
+                runtime_printer=runtime_printer,
             )
             i += 1
             continue
 
-        # TODO: maybe don't check by name for positional-only args, dunder methods
+        # TODO: maybe don't check by name for positional-only args
         # TODO: stricter checking of positional-only, keyword-only
         # TODO: check type compatibility of default args
         # TODO: overloads are sometimes pretty deceitful, so handle that better
 
+        # Allow *args and **kwargs to soak up extra args and kwargs
         stub_arg, runtime_arg = stub_args[i], runtime_args[j]
         if (stub_arg.kind == mypy.nodes.ARG_STAR) and (
             runtime_arg.kind != inspect.Parameter.VAR_POSITIONAL
@@ -266,15 +272,19 @@ def verify_funcitem(
             i += 1
             continue
 
-        if stub_arg.variable.name != runtime_arg.name:
-            # TODO: _asdict takes _self and self
+        # Ignore exact names for all dunder methods other than __init__
+        is_dunder_method = stub.name != "__init__" and stub.name.startswith("__")
+        if (
+            stub_arg.variable.name.replace("_", "") != runtime_arg.name.replace("_", "")
+            and not is_dunder_method
+        ):
             yield Error(
                 object_path,
                 f'is inconsistent, stub argument "{stub_arg.variable.name}" differs from '
                 f'runtime argument "{runtime_arg.name}"',
                 stub,
                 runtime,
-                runtime_printer=lambda s: str(inspect.signature(s)),
+                runtime_printer=runtime_printer,
             )
         i += 1
         j += 1

From 3e629ae810123631f62af1ce5152f21425fbea9b Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 17:58:53 -0800
Subject: [PATCH 28/74] stubtest: add --concise option

---
 scripts/stubtest.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 471c28395984..78a8f74c3eb1 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -68,7 +68,10 @@ def __init__(
     def is_missing_stub(self) -> bool:
         return isinstance(self.stub_object, Missing)
 
-    def __str__(self) -> str:
+    def get_description(self, concise: bool = False) -> str:
+        if concise:
+            return _style(self.object_desc, bold=True) + " " + self.message
+
         stub_line = None
         stub_file = None
         if not isinstance(self.stub_object, Missing):
@@ -362,7 +365,7 @@ def build_stubs(
     return res.files
 
 
-def main() -> Iterator[Error]:
+def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("modules", nargs="+", help="Modules to test")
     parser.add_argument(
@@ -370,6 +373,9 @@ def main() -> Iterator[Error]:
         action="store_true",
         help="Ignore errors for stub missing things that are present at runtime",
     )
+    parser.add_argument(
+        "--concise", action="store_true", help="Make output concise",
+    )
     parser.add_argument(
         "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
     )
@@ -386,9 +392,8 @@ def main() -> Iterator[Error]:
     for module in args.modules:
         for error in test_module(module, options, find_module_cache):
             if not args.ignore_missing_stub or not error.is_missing_stub():
-                yield error
+                print(error.get_description(concise=args.concise))
 
 
 if __name__ == "__main__":
-    for err in main():
-        print(err)
+    main()

From 013d04e44eecaa3ba11b0d6673ed7067036e5791 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 16 Jan 2020 18:03:32 -0800
Subject: [PATCH 29/74] stubtest: add exit code

---
 scripts/stubtest.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 78a8f74c3eb1..b6e6a13e6770 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -365,7 +365,7 @@ def build_stubs(
     return res.files
 
 
-def main() -> None:
+def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("modules", nargs="+", help="Modules to test")
     parser.add_argument(
@@ -389,11 +389,14 @@ def main() -> None:
     search_path = mypy.modulefinder.compute_search_paths([], options, data_dir)
     find_module_cache = FindModuleCache(search_path)
 
+    exit_code = 0
     for module in args.modules:
         for error in test_module(module, options, find_module_cache):
             if not args.ignore_missing_stub or not error.is_missing_stub():
+                exit_code = 1
                 print(error.get_description(concise=args.concise))
+    return exit_code
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

From 0c7a57da662cc93c3fc25647748ea8c0df79d21c Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 18 Jan 2020 14:19:15 -0800
Subject: [PATCH 30/74] stubtest: redo verify function

Rework things to avoid false positives / order nitpicks
Make checks involving *args, **kwargs a little more sophisticated
---
 scripts/stubtest.py | 227 ++++++++++++++++++++++++++++++--------------
 1 file changed, 157 insertions(+), 70 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index b6e6a13e6770..3ca8109c7ef6 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -203,9 +203,11 @@ def verify_funcitem(
     if isinstance(runtime, Missing):
         yield Error(object_path, "is not present at runtime", stub, runtime)
         return
-    if not isinstance(
-        runtime, (types.FunctionType, types.BuiltinFunctionType)
-    ) and not inspect.ismethoddescriptor(runtime):
+    if (
+        not isinstance(runtime, (types.FunctionType, types.BuiltinFunctionType))
+        and not isinstance(runtime, (types.MethodType, types.BuiltinMethodType))
+        and not inspect.ismethoddescriptor(runtime)
+    ):
         yield Error(object_path, "is not a function", stub, runtime)
         return
 
@@ -218,79 +220,162 @@ def verify_funcitem(
     def runtime_printer(s: Any) -> str:
         return "def " + str(inspect.signature(s))
 
-    i, j = 0, 0
-    stub_args = stub.arguments
-    runtime_args = list(signature.parameters.values())
-    while i < len(stub_args) or j < len(runtime_args):
-        if i >= len(stub_args):
-            # Ignore the error if the stub doesn't take **kwargs, for cases where the stub
-            # just listed out the keyword parameters the function takes
-            if runtime_args[j].kind != inspect.Parameter.VAR_KEYWORD:
-                yield Error(
-                    object_path,
-                    f'is inconsistent, stub does not have argument "{runtime_args[j].name}"',
-                    stub,
-                    runtime,
-                    runtime_printer=runtime_printer,
-                )
-            j += 1
-            continue
-        if j >= len(runtime_args):
-            yield Error(
-                object_path,
-                f"is inconsistent, runtime does not have argument {stub_args[i].variable.name}",
-                stub,
-                runtime,
-                runtime_printer=runtime_printer,
-            )
-            i += 1
-            continue
-
-        # TODO: maybe don't check by name for positional-only args
-        # TODO: stricter checking of positional-only, keyword-only
-        # TODO: check type compatibility of default args
-        # TODO: overloads are sometimes pretty deceitful, so handle that better
-
-        # Allow *args and **kwargs to soak up extra args and kwargs
-        stub_arg, runtime_arg = stub_args[i], runtime_args[j]
-        if (stub_arg.kind == mypy.nodes.ARG_STAR) and (
-            runtime_arg.kind != inspect.Parameter.VAR_POSITIONAL
-        ):
-            j += 1
-            continue
-        if (stub_arg.kind != mypy.nodes.ARG_STAR) and (
-            runtime_arg.kind == inspect.Parameter.VAR_POSITIONAL
-        ):
-            i += 1
-            continue
+    def make_error(message: str) -> Error:
+        return Error(
+            object_path,
+            "is inconsistent, " + message,
+            stub,
+            runtime,
+            runtime_printer=runtime_printer,
+        )
 
-        if (stub_arg.kind == mypy.nodes.ARG_STAR2) and (
-            runtime_arg.kind != inspect.Parameter.VAR_KEYWORD
-        ):
-            j += 1
-            continue
-        if (stub_arg.kind != mypy.nodes.ARG_STAR2) and (
-            runtime_arg.kind == inspect.Parameter.VAR_KEYWORD
+    stub_args_pos = []
+    stub_args_kwonly = {}
+    stub_args_varpos = None
+    stub_args_varkw = None
+
+    for stub_arg in stub.arguments:
+        if stub_arg.kind in (nodes.ARG_POS, nodes.ARG_OPT):
+            stub_args_pos.append(stub_arg)
+        elif stub_arg.kind in (nodes.ARG_NAMED, nodes.ARG_NAMED_OPT):
+            stub_args_kwonly[stub_arg.variable.name] = stub_arg
+        elif stub_arg.kind == nodes.ARG_STAR:
+            stub_args_varpos = stub_arg
+        elif stub_arg.kind == nodes.ARG_STAR2:
+            stub_args_varkw = stub_arg
+        else:
+            raise ValueError
+
+    runtime_args_pos = []
+    runtime_args_kwonly = {}
+    runtime_args_varpos = None
+    runtime_args_varkw = None
+
+    for runtime_arg in signature.parameters.values():
+        if runtime_arg.kind in (
+            inspect.Parameter.POSITIONAL_ONLY,
+            inspect.Parameter.POSITIONAL_OR_KEYWORD,
         ):
-            i += 1
-            continue
-
+            runtime_args_pos.append(runtime_arg)
+        elif runtime_arg.kind == inspect.Parameter.KEYWORD_ONLY:
+            runtime_args_kwonly[runtime_arg.name] = runtime_arg
+        elif runtime_arg.kind == inspect.Parameter.VAR_POSITIONAL:
+            runtime_args_varpos = runtime_arg
+        elif runtime_arg.kind == inspect.Parameter.VAR_KEYWORD:
+            runtime_args_varkw = runtime_arg
+        else:
+            raise ValueError
+
+    def verify_arg_name(
+        stub_arg: nodes.Argument, runtime_arg: inspect.Parameter
+    ) -> Iterator[Error]:
         # Ignore exact names for all dunder methods other than __init__
-        is_dunder_method = stub.name != "__init__" and stub.name.startswith("__")
+        if stub.name != "__init__" and stub.name.startswith("__"):
+            return
+        if stub_arg.variable.name.replace("_", "") != runtime_arg.name.replace("_", ""):
+            yield make_error(
+                f'stub argument "{stub_arg.variable.name}" differs from '
+                f'runtime argument "{runtime_arg.name}"'
+            )
+
+    def verify_arg_default_value(
+        stub_arg: nodes.Argument, runtime_arg: inspect.Parameter
+    ) -> Iterator[Error]:
+        if runtime_arg.default != inspect.Parameter.empty:
+            # TODO: Check that the default value is compatible with the stub type
+            if stub_arg.kind not in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
+                yield make_error(
+                    f'runtime argument "{runtime_arg.name}" has a default value '
+                    "but stub argument does not"
+                )
+        else:
+            if stub_arg.kind in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
+                yield make_error(
+                    f'stub argument "{stub_arg.variable.name}" has a default '
+                    "value but runtime argument does not"
+                )
+
+    # Check positional arguments match up
+    for stub_arg, runtime_arg in zip(stub_args_pos, runtime_args_pos):
+        yield from verify_arg_name(stub_arg, runtime_arg)
+        yield from verify_arg_default_value(stub_arg, runtime_arg)
         if (
-            stub_arg.variable.name.replace("_", "") != runtime_arg.name.replace("_", "")
-            and not is_dunder_method
+            runtime_arg.kind == inspect.Parameter.POSITIONAL_ONLY
+            and not stub_arg.variable.name.startswith("__")
+            and not stub_arg.variable.name.strip("_") == "self"
+            and not stub.name.startswith("__")  # noisy for dunder methods
+        ):
+            yield make_error(
+                f'stub argument "{stub_arg.variable.name}" should be '
+                "positional-only (rename with a leading double underscore)"
+            )
+
+    # Checks involving *args
+    if len(stub_args_pos) == len(runtime_args_pos):
+        if stub_args_varpos is None and runtime_args_varpos is not None:
+            yield make_error(
+                f'stub does not have *args argument "{runtime_args_varpos.name}"'
+            )
+        if stub_args_varpos is not None and runtime_args_varpos is None:
+            yield make_error(
+                f'runtime does not have *args argument "{stub_args_varpos.variable.name}"'
+            )
+    elif len(stub_args_pos) > len(runtime_args_pos):
+        if runtime_args_varpos is None:
+            for stub_arg in stub_args_pos[len(runtime_args_pos) :]:
+                # If the variable is in runtime_args_kwonly, it's just mislabelled as not a
+                # keyword-only argument; we report the error while checking keyword-only arguments
+                if stub_arg.variable.name not in runtime_args_kwonly:
+                    yield make_error(
+                        f'runtime does not have argument "{stub_arg.variable.name}"'
+                    )
+        # We do not check whether stub takes *args when the runtime does, for cases where the stub
+        # just listed out the extra parameters the function takes
+    elif len(stub_args_pos) < len(runtime_args_pos):
+        if stub_args_varpos is None:
+            for runtime_arg in runtime_args_pos[len(stub_args_pos) :]:
+                yield make_error(f'stub does not have argument "{runtime_arg.name}"')
+        elif runtime_args_pos is None:
+            yield make_error(
+                f'runtime does not have *args argument "{stub_args_varpos.variable.name}"'
+            )
+
+    # Check keyword-only args
+    for arg in set(stub_args_kwonly) & set(runtime_args_kwonly):
+        stub_arg, runtime_arg = stub_args_kwonly[arg], runtime_args_kwonly[arg]
+        yield from verify_arg_name(stub_arg, runtime_arg)
+        yield from verify_arg_default_value(stub_arg, runtime_arg)
+
+    # Checks involving **kwargs
+    if stub_args_varkw is None and runtime_args_varkw is not None:
+        # We do not check whether stub takes **kwargs when the runtime does, for cases where the
+        # stub just listed out the extra keyword parameters the function takes
+        # Also check against positional parameters, to avoid a nitpicky message when an argument
+        # isn't marked as keyword-only
+        stub_pos_names = set(stub_arg.variable.name for stub_arg in stub_args_pos)
+        if not set(runtime_args_kwonly).issubset(
+            set(stub_args_kwonly) | stub_pos_names
         ):
-            yield Error(
-                object_path,
-                f'is inconsistent, stub argument "{stub_arg.variable.name}" differs from '
-                f'runtime argument "{runtime_arg.name}"',
-                stub,
-                runtime,
-                runtime_printer=runtime_printer,
+            yield make_error(
+                f'stub does not have **kwargs argument "{runtime_args_varkw.name}"'
             )
-        i += 1
-        j += 1
+    if stub_args_varkw is not None and runtime_args_varkw is None:
+        yield make_error(
+            f'runtime does not have **kwargs argument "{stub_args_varkw.variable.name}"'
+        )
+    if runtime_args_varkw is None or not set(runtime_args_kwonly).issubset(
+        set(stub_args_kwonly)
+    ):
+        for arg in set(stub_args_kwonly) - set(runtime_args_kwonly):
+            yield make_error(f'runtime does not have argument "{arg}"')
+    if stub_args_varkw is None or not set(stub_args_kwonly).issubset(
+        set(runtime_args_kwonly)
+    ):
+        for arg in set(runtime_args_kwonly) - set(stub_args_kwonly):
+            if arg in set(stub_arg.variable.name for stub_arg in stub_args_pos):
+                yield make_error(f'stub argument "{arg}" is not keyword-only')
+            else:
+                yield make_error(f'stub does not have argument "{arg}"')
 
 
 @verify.register(Missing)
@@ -321,6 +406,8 @@ def verify_var(
 def verify_overloadedfuncdef(
     stub: nodes.OverloadedFuncDef, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
+    # TODO: Overloads can be pretty deceitful, so maybe be more permissive when dealing with them
+    # For a motivating example, look at RawConfigParser.items and RawConfigParser.get
     for func in stub.items:
         yield from verify(func, runtime, object_path)
 

From aac7a570a064949f8871d7abdf29ab510e411a0c Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 18 Jan 2020 14:58:08 -0800
Subject: [PATCH 31/74] stubtest: add ability to whitelist errors

---
 scripts/stubtest.py | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 3ca8109c7ef6..656a0de6f8e8 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -455,16 +455,25 @@ def build_stubs(
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("modules", nargs="+", help="Modules to test")
+    parser.add_argument(
+        "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
+    )
     parser.add_argument(
         "--ignore-missing-stub",
         action="store_true",
         help="Ignore errors for stub missing things that are present at runtime",
     )
+    parser.add_argument(
+        "--whitelist",
+        help="Use file as a whitelist. Whitelists can be created with --output-whitelist",
+    )
     parser.add_argument(
         "--concise", action="store_true", help="Make output concise",
     )
     parser.add_argument(
-        "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
+        "--output-whitelist",
+        action="store_true",
+        help="Print a whitelist (to stdout) to be used with --whitelist",
     )
     args = parser.parse_args()
 
@@ -476,12 +485,23 @@ def main() -> int:
     search_path = mypy.modulefinder.compute_search_paths([], options, data_dir)
     find_module_cache = FindModuleCache(search_path)
 
+    whitelist = set()
+    if args.whitelist:
+        with open(args.whitelist) as f:
+            whitelist = set(l.strip() for l in f.readlines())
+
     exit_code = 0
     for module in args.modules:
         for error in test_module(module, options, find_module_cache):
-            if not args.ignore_missing_stub or not error.is_missing_stub():
-                exit_code = 1
-                print(error.get_description(concise=args.concise))
+            if args.ignore_missing_stub and error.is_missing_stub():
+                continue
+            if error.object_desc in whitelist:
+                continue
+            if args.output_whitelist:
+                print(error.object_desc)
+                continue
+            exit_code = 1
+            print(error.get_description(concise=args.concise))
     return exit_code
 
 

From 9a680627d2482b3aaac6194228722ada6c0096e1 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 18 Jan 2020 16:33:14 -0800
Subject: [PATCH 32/74] stubtest: [minor] clean up error handling

---
 scripts/stubtest.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 656a0de6f8e8..2def25ba0891 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -18,7 +18,6 @@
 import mypy.modulefinder
 import mypy.types
 from mypy import nodes
-from mypy.errors import CompileError
 from mypy.modulefinder import FindModuleCache
 from mypy.options import Options
 from mypy.util import FancyFormatter
@@ -129,8 +128,15 @@ def test_module(
         if (mod == module_name or mod.startswith(module_name + "."))
     }
 
+    if not stubs:
+        yield Error([module_name], "failed to find stubs", MISSING, None)
+
     for mod, stub in stubs.items():
-        runtime = importlib.import_module(mod)
+        try:
+            runtime = importlib.import_module(mod)
+        except Exception as e:
+            yield Error([mod], f"failed to import: {e}", stub, MISSING)
+            continue
         yield from verify(stub, runtime, [mod])
 
 
@@ -244,7 +250,7 @@ def make_error(message: str) -> Error:
         elif stub_arg.kind == nodes.ARG_STAR2:
             stub_args_varkw = stub_arg
         else:
-            raise ValueError
+            assert False
 
     runtime_args_pos = []
     runtime_args_kwonly = {}
@@ -264,7 +270,7 @@ def make_error(message: str) -> Error:
         elif runtime_arg.kind == inspect.Parameter.VAR_KEYWORD:
             runtime_args_varkw = runtime_arg
         else:
-            raise ValueError
+            assert False
 
     def verify_arg_name(
         stub_arg: nodes.Argument, runtime_arg: inspect.Parameter
@@ -383,8 +389,7 @@ def verify_none(
     stub: Missing, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
     yield Error(object_path, "is not present in stub", stub, runtime)
-    if isinstance(runtime, Missing):
-        raise RuntimeError
+    assert not isinstance(runtime, Missing)
 
 
 @verify.register(nodes.Var)
@@ -447,8 +452,13 @@ def build_stubs(
 
     res = mypy.build.build(sources=sources, options=options)
     if res.errors:
-        raise CompileError
-
+        output = [
+            _style("error: ", color="red", bold=True),
+            _style(module_name, bold=True),
+            " failed mypy build.\n",
+        ]
+        print("".join(output) + "\n".join(res.errors))
+        sys.exit(1)
     return res.files
 
 

From 871be3eeb97fefb9d960d790a4671bac554d7507 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 18 Jan 2020 17:13:45 -0800
Subject: [PATCH 33/74] stubtest: add --check-typeshed option

---
 scripts/stubtest.py | 57 +++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 2 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 2def25ba0891..8ed85e61fb96 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -7,9 +7,11 @@
 import argparse
 import importlib
 import inspect
+import subprocess
 import sys
 import types
 from functools import singledispatch
+from pathlib import Path
 from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, Union
 
 from typing_extensions import Type
@@ -462,9 +464,49 @@ def build_stubs(
     return res.files
 
 
+def get_typeshed_stdlib_modules(
+    data_dir: str, custom_typeshed_dir: Optional[str]
+) -> List[str]:
+    # This snippet is based on code in mypy.modulefinder.default_lib_path
+    if custom_typeshed_dir:
+        typeshed_dir = Path(custom_typeshed_dir)
+    else:
+        typeshed_dir = Path(data_dir)
+        if (typeshed_dir / "stubs-auto").exists():
+            typeshed_dir /= "stubs-auto"
+        typeshed_dir /= "typeshed"
+
+    versions = ["2and3", "3"]
+    for minor in range(sys.version_info.minor + 1):
+        versions.append(f"3.{minor}")
+
+    modules = []
+    for version in versions:
+        base = typeshed_dir / "stdlib" / version
+        if base.exists():
+            output = subprocess.check_output(
+                ["find", base, "-type", "f"], encoding="utf-8"
+            )
+            paths = [Path(p) for p in output.splitlines()]
+            for path in paths:
+                if path.stem == "__init__":
+                    path = path.parent
+                modules.append(
+                    ".".join(path.relative_to(base).parts[:-1] + (path.stem,))
+                )
+    return sorted(modules)
+
+
 def main() -> int:
+    assert sys.version_info >= (3, 6), "This script requires at least Python 3.6"
+
     parser = argparse.ArgumentParser()
-    parser.add_argument("modules", nargs="+", help="Modules to test")
+    parser.add_argument("modules", nargs="*", help="Modules to test")
+    parser.add_argument(
+        "--check-typeshed",
+        action="store_true",
+        help="Check all stdlib modules in typeshed",
+    )
     parser.add_argument(
         "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
     )
@@ -500,8 +542,19 @@ def main() -> int:
         with open(args.whitelist) as f:
             whitelist = set(l.strip() for l in f.readlines())
 
+    modules = args.modules
+    if args.check_typeshed:
+        assert (
+            not args.modules
+        ), "Cannot pass both --check-typeshed and a list of modules"
+        modules = get_typeshed_stdlib_modules(data_dir, args.custom_typeshed_dir)
+        # TODO: See if there's a more efficient way to get mypy to build all the stubs, rather than
+        # just one by one
+
+    assert modules, "No modules to check"
+
     exit_code = 0
-    for module in args.modules:
+    for module in modules:
         for error in test_module(module, options, find_module_cache):
             if args.ignore_missing_stub and error.is_missing_stub():
                 continue

From ae96f92845eff673345c59b0089f7898a942d78e Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 18 Jan 2020 22:26:05 -0800
Subject: [PATCH 34/74] stubtest: [minor] handle distutils.command a little
 better

---
 scripts/stubtest.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 8ed85e61fb96..e08788c8cc0a 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -390,8 +390,15 @@ def verify_arg_default_value(
 def verify_none(
     stub: Missing, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
+    if isinstance(runtime, Missing):
+        try:
+            # We shouldn't really get here, however, some modules like distutils.command have some
+            # weird things going on. Try to see if we can find a runtime object by importing it,
+            # otherwise crash.
+            runtime = importlib.import_module(".".join(object_path))
+        except ModuleNotFoundError:
+            assert False
     yield Error(object_path, "is not present in stub", stub, runtime)
-    assert not isinstance(runtime, Missing)
 
 
 @verify.register(nodes.Var)

From 412fcf3c4378fc48b638809747bdc8986ec02175 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sun, 19 Jan 2020 00:55:14 -0600
Subject: [PATCH 35/74] stubtest: adjust module level things we check in stubs

---
 scripts/stubtest.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index e08788c8cc0a..9bff2be24e2b 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -162,15 +162,19 @@ def verify_mypyfile(
         yield Error(object_path, "is not a module", stub, runtime)
         return
 
-    # Check all things in the stub
-    to_check = set(m for m, o in stub.names.items() if o.module_public)
+    # Check things in the stub that are public
+    to_check = set(
+        m
+        for m, o in stub.names.items()
+        if o.module_public and (not m.startswith("_") or hasattr(runtime, m))
+    )
     # Check all things declared in module's __all__
     to_check.update(getattr(runtime, "__all__", []))
     to_check.difference_update(
         {"__file__", "__doc__", "__name__", "__builtins__", "__package__"}
     )
     # We currently don't check things in the module that aren't in the stub, other than things that
-    # are in __all__ to avoid false positives.
+    # are in __all__, to avoid false positives.
 
     for entry in sorted(to_check):
         yield from verify(

From 80b3d26e4136b188a7eeba094cf4ebdd7e28fb79 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sun, 19 Jan 2020 01:12:37 -0600
Subject: [PATCH 36/74] stubtest: check for mistaken positional only args

---
 scripts/stubtest.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 9bff2be24e2b..87159655551f 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -284,6 +284,7 @@ def verify_arg_name(
         # Ignore exact names for all dunder methods other than __init__
         if stub.name != "__init__" and stub.name.startswith("__"):
             return
+        # TODO: make this check a little less hacky, maybe be more lax for positional-only args
         if stub_arg.variable.name.replace("_", "") != runtime_arg.name.replace("_", ""):
             yield make_error(
                 f'stub argument "{stub_arg.variable.name}" differs from '
@@ -321,6 +322,14 @@ def verify_arg_default_value(
                 f'stub argument "{stub_arg.variable.name}" should be '
                 "positional-only (rename with a leading double underscore)"
             )
+        if (
+            runtime_arg.kind != inspect.Parameter.POSITIONAL_ONLY
+            and stub_arg.variable.name.startswith("__")
+        ):
+            yield make_error(
+                f'stub argument "{stub_arg.variable.name}" is positional or keyword '
+                "(remove leading double underscore)"
+            )
 
     # Checks involving *args
     if len(stub_args_pos) == len(runtime_args_pos):

From 59a38b9f4fad69e81ef3d6c4b9ee066be13bea4c Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sun, 19 Jan 2020 16:51:21 -0600
Subject: [PATCH 37/74] stubtest: be more permissive about positional-only arg
 names

---
 scripts/stubtest.py | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 87159655551f..a758f1c518b4 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -284,12 +284,31 @@ def verify_arg_name(
         # Ignore exact names for all dunder methods other than __init__
         if stub.name != "__init__" and stub.name.startswith("__"):
             return
-        # TODO: make this check a little less hacky, maybe be more lax for positional-only args
-        if stub_arg.variable.name.replace("_", "") != runtime_arg.name.replace("_", ""):
-            yield make_error(
-                f'stub argument "{stub_arg.variable.name}" differs from '
-                f'runtime argument "{runtime_arg.name}"'
-            )
+
+        def strip_prefix(s: str, prefix: str) -> str:
+            return s[len(prefix) :] if s.startswith(prefix) else s
+
+        if strip_prefix(stub_arg.variable.name, "__") == runtime_arg.name:
+            return
+
+        def names_approx_match(a: str, b: str) -> bool:
+            a = a.strip("_")
+            b = b.strip("_")
+            return a.startswith(b) or b.startswith(a) or len(a) == 1 or len(b) == 1
+
+        # Be more permissive about names matching for positional-only arguments
+        if (
+            runtime_arg.kind == inspect.Parameter.POSITIONAL_ONLY
+            and names_approx_match(stub_arg.variable.name, runtime_arg.name)
+        ):
+            return
+        # This comes up with namedtuples, so ignore
+        if stub_arg.variable.name == "_self":
+            return
+        yield make_error(
+            f'stub argument "{stub_arg.variable.name}" differs from '
+            f'runtime argument "{runtime_arg.name}"'
+        )
 
     def verify_arg_default_value(
         stub_arg: nodes.Argument, runtime_arg: inspect.Parameter

From b0f226cf2d63823b5c15d4f8cb45e27b468cd64d Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sun, 19 Jan 2020 23:28:15 -0600
Subject: [PATCH 38/74] stubtest: [minor] make error order more deterministic

---
 scripts/stubtest.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index a758f1c518b4..2a20a6910ce1 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -381,7 +381,7 @@ def verify_arg_default_value(
             )
 
     # Check keyword-only args
-    for arg in set(stub_args_kwonly) & set(runtime_args_kwonly):
+    for arg in sorted(set(stub_args_kwonly) & set(runtime_args_kwonly)):
         stub_arg, runtime_arg = stub_args_kwonly[arg], runtime_args_kwonly[arg]
         yield from verify_arg_name(stub_arg, runtime_arg)
         yield from verify_arg_default_value(stub_arg, runtime_arg)
@@ -406,12 +406,12 @@ def verify_arg_default_value(
     if runtime_args_varkw is None or not set(runtime_args_kwonly).issubset(
         set(stub_args_kwonly)
     ):
-        for arg in set(stub_args_kwonly) - set(runtime_args_kwonly):
+        for arg in sorted(set(stub_args_kwonly) - set(runtime_args_kwonly)):
             yield make_error(f'runtime does not have argument "{arg}"')
     if stub_args_varkw is None or not set(stub_args_kwonly).issubset(
         set(runtime_args_kwonly)
     ):
-        for arg in set(runtime_args_kwonly) - set(stub_args_kwonly):
+        for arg in sorted(set(runtime_args_kwonly) - set(stub_args_kwonly)):
             if arg in set(stub_arg.variable.name for stub_arg in stub_args_pos):
                 yield make_error(f'stub argument "{arg}" is not keyword-only')
             else:

From b1139ea99b5dc305353ec161a8f29b433956da82 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Mon, 20 Jan 2020 13:41:49 -0600
Subject: [PATCH 39/74] stubtest: only mypy build once

This makes the script complete 100x faster when using --check-typeshed
---
 scripts/stubtest.py | 84 ++++++++++++++++++++++++---------------------
 1 file changed, 44 insertions(+), 40 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 2a20a6910ce1..2522d0722b31 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -20,7 +20,6 @@
 import mypy.modulefinder
 import mypy.types
 from mypy import nodes
-from mypy.modulefinder import FindModuleCache
 from mypy.options import Options
 from mypy.util import FancyFormatter
 
@@ -121,25 +120,19 @@ def get_description(self, concise: bool = False) -> str:
         return "".join(output)
 
 
-def test_module(
-    module_name: str, options: Options, find_module_cache: FindModuleCache
-) -> Iterator[Error]:
-    stubs = {
-        mod: stub
-        for mod, stub in build_stubs(module_name, options, find_module_cache).items()
-        if (mod == module_name or mod.startswith(module_name + "."))
-    }
-
-    if not stubs:
+def test_module(module_name: str) -> Iterator[Error]:
+    stub = get_stub(module_name)
+    if stub is None:
         yield Error([module_name], "failed to find stubs", MISSING, None)
+        return
 
-    for mod, stub in stubs.items():
-        try:
-            runtime = importlib.import_module(mod)
-        except Exception as e:
-            yield Error([mod], f"failed to import: {e}", stub, MISSING)
-            continue
-        yield from verify(stub, runtime, [mod])
+    try:
+        runtime = importlib.import_module(module_name)
+    except Exception as e:
+        yield Error([module_name], f"failed to import: {e}", stub, MISSING)
+        return
+
+    yield from verify(stub, runtime, [module_name])
 
 
 @singledispatch
@@ -486,31 +479,46 @@ def verify_typealias(
         yield None
 
 
-def build_stubs(
-    module_name: str, options: Options, find_module_cache: FindModuleCache
-) -> Dict[str, nodes.MypyFile]:
-    sources = find_module_cache.find_modules_recursive(module_name)
+_all_stubs: Dict[str, nodes.MypyFile] = {}
+
+
+def build_stubs(modules: List[str], options: Options) -> None:
+    data_dir = mypy.build.default_data_dir()
+    search_path = mypy.modulefinder.compute_search_paths([], options, data_dir)
+    find_module_cache = mypy.modulefinder.FindModuleCache(search_path)
+
+    sources = []
+    # TODO: restore support for automatically recursing into submodules with find_modules_recursive
+    for module in modules:
+        module_path = find_module_cache.find_module(module)
+        if module_path is None:
+            # test_module will yield an error later when it can't find stubs
+            continue
+        sources.append(mypy.modulefinder.BuildSource(module_path, module, None))
 
     res = mypy.build.build(sources=sources, options=options)
     if res.errors:
         output = [
             _style("error: ", color="red", bold=True),
-            _style(module_name, bold=True),
             " failed mypy build.\n",
         ]
         print("".join(output) + "\n".join(res.errors))
         sys.exit(1)
-    return res.files
 
+    global _all_stubs
+    _all_stubs = res.files
 
-def get_typeshed_stdlib_modules(
-    data_dir: str, custom_typeshed_dir: Optional[str]
-) -> List[str]:
+
+def get_stub(module: str) -> Optional[nodes.MypyFile]:
+    return _all_stubs.get(module)
+
+
+def get_typeshed_stdlib_modules(custom_typeshed_dir: Optional[str]) -> List[str]:
     # This snippet is based on code in mypy.modulefinder.default_lib_path
     if custom_typeshed_dir:
         typeshed_dir = Path(custom_typeshed_dir)
     else:
-        typeshed_dir = Path(data_dir)
+        typeshed_dir = Path(mypy.build.default_data_dir())
         if (typeshed_dir / "stubs-auto").exists():
             typeshed_dir /= "stubs-auto"
         typeshed_dir /= "typeshed"
@@ -568,14 +576,6 @@ def main() -> int:
     )
     args = parser.parse_args()
 
-    options = Options()
-    options.incremental = False
-    options.custom_typeshed_dir = args.custom_typeshed_dir
-
-    data_dir = mypy.build.default_data_dir()
-    search_path = mypy.modulefinder.compute_search_paths([], options, data_dir)
-    find_module_cache = FindModuleCache(search_path)
-
     whitelist = set()
     if args.whitelist:
         with open(args.whitelist) as f:
@@ -586,15 +586,19 @@ def main() -> int:
         assert (
             not args.modules
         ), "Cannot pass both --check-typeshed and a list of modules"
-        modules = get_typeshed_stdlib_modules(data_dir, args.custom_typeshed_dir)
-        # TODO: See if there's a more efficient way to get mypy to build all the stubs, rather than
-        # just one by one
+        modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir)
 
     assert modules, "No modules to check"
 
+    options = Options()
+    options.incremental = False
+    options.custom_typeshed_dir = args.custom_typeshed_dir
+
+    build_stubs(modules, options)
+
     exit_code = 0
     for module in modules:
-        for error in test_module(module, options, find_module_cache):
+        for error in test_module(module):
             if args.ignore_missing_stub and error.is_missing_stub():
                 continue
             if error.object_desc in whitelist:

From 141e80002b50c002d9862e782a51808dee254200 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Mon, 20 Jan 2020 13:45:50 -0600
Subject: [PATCH 40/74] stubtest: [minor] remove antigravity from
 --check-typeshed

---
 scripts/stubtest.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 2522d0722b31..f36caadc430f 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -587,6 +587,7 @@ def main() -> int:
             not args.modules
         ), "Cannot pass both --check-typeshed and a list of modules"
         modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir)
+        modules.remove("antigravity")  # it's super annoying
 
     assert modules, "No modules to check"
 

From f4b1a2a8e79fc33da3cf6e2d4c89ab8e0f2b4133 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Mon, 20 Jan 2020 14:12:08 -0600
Subject: [PATCH 41/74] stubtest: make verify_var work

---
 scripts/stubtest.py | 55 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 51 insertions(+), 4 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index f36caadc430f..f1b2a5030896 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -435,10 +435,19 @@ def verify_var(
         if len(object_path) <= 1:
             yield Error(object_path, "is not present at runtime", stub, runtime)
         return
-    # TODO: Make this better
-    if isinstance(stub, mypy.types.Instance):
-        if stub.type.type.name != runtime.__name__:
-            yield Error(object_path, "var_mismatch", stub, runtime)
+
+    runtime_type = get_mypy_type_of_runtime_value(runtime)
+    if (
+        runtime_type is not None
+        and stub.type is not None
+        and not is_subtype_helper(runtime_type, stub.type)
+    ):
+        yield Error(
+            object_path,
+            f"variable differs from runtime type {runtime_type}",
+            stub,
+            runtime,
+        )
 
 
 @verify.register(nodes.OverloadedFuncDef)
@@ -479,6 +488,44 @@ def verify_typealias(
         yield None
 
 
+def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool:
+    with mypy.state.strict_optional_set(True):
+        return mypy.subtypes.is_subtype(left, right)
+
+
+def get_mypy_type_of_runtime_value(runtime: Any) -> Optional[mypy.types.Type]:
+    if runtime is None:
+        return mypy.types.NoneType()
+    if isinstance(runtime, property):
+        return None
+    if isinstance(runtime, (types.FunctionType, types.BuiltinFunctionType)):
+        # TODO: Construct a mypy.types.CallableType
+        return None
+    stub = get_stub(type(runtime).__module__)
+    if stub is not None:
+        type_name = type(runtime).__name__
+        if type_name in stub.names:
+            type_info = stub.names[type_name].node
+            if isinstance(type_info, nodes.TypeInfo):
+                anytype = lambda: mypy.types.AnyType(mypy.types.TypeOfAny.unannotated)
+
+                if isinstance(runtime, tuple):
+                    opt_items = [get_mypy_type_of_runtime_value(v) for v in runtime]
+                    items = [(i if i is not None else anytype()) for i in opt_items]
+                    fallback = mypy.types.Instance(type_info, [anytype()])
+                    return mypy.types.TupleType(items, fallback)
+
+                # Technically, Literals are supposed to be only bool, int, str or bytes, but this
+                # seems to work fine
+                return mypy.types.LiteralType(
+                    value=runtime,
+                    fallback=mypy.types.Instance(
+                        type_info, [anytype() for _ in type_info.type_vars]
+                    ),
+                )
+    return None
+
+
 _all_stubs: Dict[str, nodes.MypyFile] = {}
 
 

From 22c4cf258c8cd8bcdb3ce136f19daa6530a0de45 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Mon, 20 Jan 2020 19:36:06 -0600
Subject: [PATCH 42/74] stubtest: verify types of argument default values

---
 scripts/stubtest.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index f1b2a5030896..adfc1e263566 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -307,12 +307,25 @@ def verify_arg_default_value(
         stub_arg: nodes.Argument, runtime_arg: inspect.Parameter
     ) -> Iterator[Error]:
         if runtime_arg.default != inspect.Parameter.empty:
-            # TODO: Check that the default value is compatible with the stub type
             if stub_arg.kind not in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
                 yield make_error(
                     f'runtime argument "{runtime_arg.name}" has a default value '
                     "but stub argument does not"
                 )
+            else:
+                runtime_type = get_mypy_type_of_runtime_value(runtime_arg.default)
+                if (
+                    runtime_type is not None
+                    and stub_arg.variable.type is not None
+                    # Avoid false positives for marker objects
+                    and type(runtime_arg.default) != object
+                    and not is_subtype_helper(runtime_type, stub_arg.variable.type)
+                ):
+                    yield make_error(
+                        f'runtime argument "{runtime_arg.name}" has a default value of type '
+                        f"{runtime_type}, which is incompatible with stub argument type "
+                        f"{stub_arg.variable.type}"
+                    )
         else:
             if stub_arg.kind in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
                 yield make_error(

From 8481380477f64f9fb02cb0efc5df9c4c669c92b7 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Mon, 20 Jan 2020 19:44:27 -0600
Subject: [PATCH 43/74] stubtest: pretend Literal[0, 1] is subtype of bool

---
 scripts/stubtest.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index adfc1e263566..33fae2bae8e0 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -502,6 +502,15 @@ def verify_typealias(
 
 
 def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool:
+    if (
+        isinstance(left, mypy.types.LiteralType)
+        and isinstance(left.value, int)
+        and left.value in (0, 1)
+        and isinstance(right, mypy.types.Instance)
+        and right.type.fullname == "builtins.bool"
+    ):
+        # Pretend Literal[0, 1] is a subtype of bool to avoid unhelpful errors.
+        return True
     with mypy.state.strict_optional_set(True):
         return mypy.subtypes.is_subtype(left, right)
 

From 2568583d9f6dcc792182ad9216639c872e97493e Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Mon, 20 Jan 2020 22:12:01 -0600
Subject: [PATCH 44/74] stubtest: output unused whitelist entries

---
 scripts/stubtest.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 33fae2bae8e0..5cb3e6539982 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -645,10 +645,10 @@ def main() -> int:
     )
     args = parser.parse_args()
 
-    whitelist = set()
+    whitelist = {}
     if args.whitelist:
         with open(args.whitelist) as f:
-            whitelist = set(l.strip() for l in f.readlines())
+            whitelist = {l.strip(): False for l in f.readlines()}
 
     modules = args.modules
     if args.check_typeshed:
@@ -672,12 +672,19 @@ def main() -> int:
             if args.ignore_missing_stub and error.is_missing_stub():
                 continue
             if error.object_desc in whitelist:
+                whitelist[error.object_desc] = True
                 continue
             if args.output_whitelist:
                 print(error.object_desc)
                 continue
             exit_code = 1
             print(error.get_description(concise=args.concise))
+
+    for w in whitelist:
+        if not whitelist[w]:
+            exit_code = 1
+            print(f"note: unused whitelist entry {w}")
+
     return exit_code
 
 

From 2de4facaaedad92e2f8716ddafa831e3bd968b42 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Mon, 20 Jan 2020 22:16:09 -0600
Subject: [PATCH 45/74] stubtest: [minor] deduplicate, sort --output-whitelist,
 fix exit code

---
 scripts/stubtest.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 5cb3e6539982..a7afb1ba4734 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -650,6 +650,8 @@ def main() -> int:
         with open(args.whitelist) as f:
             whitelist = {l.strip(): False for l in f.readlines()}
 
+    output_whitelist = set()
+
     modules = args.modules
     if args.check_typeshed:
         assert (
@@ -674,10 +676,11 @@ def main() -> int:
             if error.object_desc in whitelist:
                 whitelist[error.object_desc] = True
                 continue
+
+            exit_code = 1
             if args.output_whitelist:
-                print(error.object_desc)
+                output_whitelist.add(error.object_desc)
                 continue
-            exit_code = 1
             print(error.get_description(concise=args.concise))
 
     for w in whitelist:
@@ -685,6 +688,11 @@ def main() -> int:
             exit_code = 1
             print(f"note: unused whitelist entry {w}")
 
+    if args.output_whitelist:
+        for e in sorted(output_whitelist):
+            print(e)
+        exit_code = 0
+
     return exit_code
 
 

From b778b44ae2b15c3ca6ba8d69ddd50e5564f5fcc5 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 21 Jan 2020 01:17:24 -0600
Subject: [PATCH 46/74] stubtest: add more documentation

Also flatten out get_mypy_type_of_runtime_value
---
 scripts/stubtest.py | 115 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 92 insertions(+), 23 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index a7afb1ba4734..0c256ac06a35 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -25,6 +25,8 @@
 
 
 class Missing:
+    """Marker object for things that are missing (from a stub or the runtime)."""
+
     def __repr__(self) -> str:
         return "MISSING"
 
@@ -38,6 +40,7 @@ def __repr__(self) -> str:
 
 
 def _style(message: str, **kwargs: Any) -> str:
+    """Wrapper around mypy.util for fancy formatting."""
     kwargs.setdefault("color", "none")
     return _formatter.style(message, **kwargs)
 
@@ -52,6 +55,16 @@ def __init__(
         stub_printer: Optional[Callable[[nodes.Node], str]] = None,
         runtime_printer: Optional[Callable[[Any], str]] = None,
     ) -> None:
+        """Represents an error found by stubtest.
+
+        :param object_path: Location of the object with the error, eg, ["module", "Class", "method"]
+        :param message: Error message
+        :param stub_object: The mypy node representing the stub
+        :param runtime_object: Actual object obtained from the runtime
+        :param stub_printer: Callable to provide specialised output for a given stub object
+        :param runtime_printer: Callable to provide specialised output for a given runtime object
+
+        """
         self.object_desc = ".".join(object_path)
         self.message = message
         self.stub_object = stub_object
@@ -66,9 +79,15 @@ def __init__(
         )
 
     def is_missing_stub(self) -> bool:
+        """Whether or not the error is for something missing from the stub."""
         return isinstance(self.stub_object, Missing)
 
     def get_description(self, concise: bool = False) -> str:
+        """Returns a description of the error.
+
+        :param concise: Whether to return a concise, one-line description
+
+        """
         if concise:
             return _style(self.object_desc, bold=True) + " " + self.message
 
@@ -121,6 +140,13 @@ def get_description(self, concise: bool = False) -> str:
 
 
 def test_module(module_name: str) -> Iterator[Error]:
+    """Tests a given module's stub against introspecting it at runtime.
+
+    Requires the stub to have been built already, accomplished by a call to ``build_stubs``.
+
+    :param module_name: The module to test
+
+    """
     stub = get_stub(module_name)
     if stub is None:
         yield Error([module_name], "failed to find stubs", MISSING, None)
@@ -139,6 +165,14 @@ def test_module(module_name: str) -> Iterator[Error]:
 def verify(
     stub: nodes.Node, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
+    """Entry point for comparing a stub to a runtime object.
+
+    We use single dispatch based on the type of ``stub``.
+
+    :param stub: The mypy node representing a part of the stub
+    :param runtime: The runtime object corresponding to ``stub``
+
+    """
     yield Error(object_path, "is an unknown mypy node", stub, runtime)
 
 
@@ -234,6 +268,7 @@ def make_error(message: str) -> Error:
             runtime_printer=runtime_printer,
         )
 
+    # Extract various arguments by type from the stub
     stub_args_pos = []
     stub_args_kwonly = {}
     stub_args_varpos = None
@@ -251,6 +286,7 @@ def make_error(message: str) -> Error:
         else:
             assert False
 
+    # Extract various arguments by type from the runtime object
     runtime_args_pos = []
     runtime_args_kwonly = {}
     runtime_args_varpos = None
@@ -274,6 +310,7 @@ def make_error(message: str) -> Error:
     def verify_arg_name(
         stub_arg: nodes.Argument, runtime_arg: inspect.Parameter
     ) -> Iterator[Error]:
+        """Checks whether argument names match."""
         # Ignore exact names for all dunder methods other than __init__
         if stub.name != "__init__" and stub.name.startswith("__"):
             return
@@ -306,6 +343,7 @@ def names_approx_match(a: str, b: str) -> bool:
     def verify_arg_default_value(
         stub_arg: nodes.Argument, runtime_arg: inspect.Parameter
     ) -> Iterator[Error]:
+        """Checks whether argument default values are compatible."""
         if runtime_arg.default != inspect.Parameter.empty:
             if stub_arg.kind not in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
                 yield make_error(
@@ -430,7 +468,8 @@ def verify_none(
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
         try:
-            # We shouldn't really get here, however, some modules like distutils.command have some
+            # We shouldn't really get here since that would involve something not existing both in
+            # the stub and the runtime, however, some modules like distutils.command have some
             # weird things going on. Try to see if we can find a runtime object by importing it,
             # otherwise crash.
             runtime = importlib.import_module(".".join(object_path))
@@ -490,8 +529,11 @@ def verify_decorator(
         and isinstance(stub.decorators[0], nodes.NameExpr)
         and stub.decorators[0].fullname == "typing.overload"
     ):
+        # If the only decorator is @typing.overload, just delegate to the usual verify_funcitem
         yield from verify(stub.func, runtime, object_path)
 
+    # TODO: See if there are other common decorators we should be checking
+
 
 @verify.register(nodes.TypeAlias)
 def verify_typealias(
@@ -502,6 +544,7 @@ def verify_typealias(
 
 
 def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool:
+    """Checks whether ``left`` is a subtype of ``right``."""
     if (
         isinstance(left, mypy.types.LiteralType)
         and isinstance(left.value, int)
@@ -516,42 +559,59 @@ def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool:
 
 
 def get_mypy_type_of_runtime_value(runtime: Any) -> Optional[mypy.types.Type]:
+    """Returns a mypy type object representing the type of ``runtime``.
+
+    Returns None if we can't find something that works.
+
+    """
     if runtime is None:
         return mypy.types.NoneType()
     if isinstance(runtime, property):
+        # Give up on properties to avoid issues with things that are typed as attributes.
         return None
     if isinstance(runtime, (types.FunctionType, types.BuiltinFunctionType)):
         # TODO: Construct a mypy.types.CallableType
         return None
+
+    # Try and look up a stub for the runtime object
     stub = get_stub(type(runtime).__module__)
-    if stub is not None:
-        type_name = type(runtime).__name__
-        if type_name in stub.names:
-            type_info = stub.names[type_name].node
-            if isinstance(type_info, nodes.TypeInfo):
-                anytype = lambda: mypy.types.AnyType(mypy.types.TypeOfAny.unannotated)
-
-                if isinstance(runtime, tuple):
-                    opt_items = [get_mypy_type_of_runtime_value(v) for v in runtime]
-                    items = [(i if i is not None else anytype()) for i in opt_items]
-                    fallback = mypy.types.Instance(type_info, [anytype()])
-                    return mypy.types.TupleType(items, fallback)
-
-                # Technically, Literals are supposed to be only bool, int, str or bytes, but this
-                # seems to work fine
-                return mypy.types.LiteralType(
-                    value=runtime,
-                    fallback=mypy.types.Instance(
-                        type_info, [anytype() for _ in type_info.type_vars]
-                    ),
-                )
-    return None
+    if stub is None:
+        return None
+    type_name = type(runtime).__name__
+    if type_name not in stub.names:
+        return None
+    type_info = stub.names[type_name].node
+    if not isinstance(type_info, nodes.TypeInfo):
+        return None
+
+    anytype = lambda: mypy.types.AnyType(mypy.types.TypeOfAny.unannotated)
+
+    if isinstance(runtime, tuple):
+        # Special case tuples so we construct a valid mypy.types.TupleType
+        opt_items = [get_mypy_type_of_runtime_value(v) for v in runtime]
+        items = [(i if i is not None else anytype()) for i in opt_items]
+        fallback = mypy.types.Instance(type_info, [anytype()])
+        return mypy.types.TupleType(items, fallback)
+
+    # Technically, Literals are supposed to be only bool, int, str or bytes, but this
+    # seems to work fine
+    return mypy.types.LiteralType(
+        value=runtime,
+        fallback=mypy.types.Instance(
+            type_info, [anytype() for _ in type_info.type_vars]
+        ),
+    )
 
 
 _all_stubs: Dict[str, nodes.MypyFile] = {}
 
 
 def build_stubs(modules: List[str], options: Options) -> None:
+    """Uses mypy to construct stub objects for the given modules.
+
+    This sets global state that ``get_stub`` can access.
+
+    """
     data_dir = mypy.build.default_data_dir()
     search_path = mypy.modulefinder.compute_search_paths([], options, data_dir)
     find_module_cache = mypy.modulefinder.FindModuleCache(search_path)
@@ -579,10 +639,12 @@ def build_stubs(modules: List[str], options: Options) -> None:
 
 
 def get_stub(module: str) -> Optional[nodes.MypyFile]:
+    """Returns a stub object for the given module, if we've built one."""
     return _all_stubs.get(module)
 
 
 def get_typeshed_stdlib_modules(custom_typeshed_dir: Optional[str]) -> List[str]:
+    """Returns a list of stdlib modules in typeshed (for current Python version)."""
     # This snippet is based on code in mypy.modulefinder.default_lib_path
     if custom_typeshed_dir:
         typeshed_dir = Path(custom_typeshed_dir)
@@ -647,9 +709,12 @@ def main() -> int:
 
     whitelist = {}
     if args.whitelist:
+        # Load the whitelist. This is a series of strings corresponding to Error.object_desc
+        # Values in the dict will store whether we used the whitelist entry or not.
         with open(args.whitelist) as f:
             whitelist = {l.strip(): False for l in f.readlines()}
 
+    # If we need to output a whitelist, we store Error.object_desc for each error here.
     output_whitelist = set()
 
     modules = args.modules
@@ -671,23 +736,27 @@ def main() -> int:
     exit_code = 0
     for module in modules:
         for error in test_module(module):
+            # Filter errors
             if args.ignore_missing_stub and error.is_missing_stub():
                 continue
             if error.object_desc in whitelist:
                 whitelist[error.object_desc] = True
                 continue
 
+            # We have errors, so change exit code, and output whatever necessary
             exit_code = 1
             if args.output_whitelist:
                 output_whitelist.add(error.object_desc)
                 continue
             print(error.get_description(concise=args.concise))
 
+    # Print unused whitelist entries
     for w in whitelist:
         if not whitelist[w]:
             exit_code = 1
             print(f"note: unused whitelist entry {w}")
 
+    # Print the generated whitelist
     if args.output_whitelist:
         for e in sorted(output_whitelist):
             print(e)

From 8bebf33d98d9104993b0195f2c3bc0fad91aee9a Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 21 Jan 2020 01:18:27 -0600
Subject: [PATCH 47/74] stubtest: [minor] rename --output-whitelist to
 --generate-whitelist

---
 scripts/stubtest.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 0c256ac06a35..685b7be7ae36 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -695,13 +695,13 @@ def main() -> int:
     )
     parser.add_argument(
         "--whitelist",
-        help="Use file as a whitelist. Whitelists can be created with --output-whitelist",
+        help="Use file as a whitelist. Whitelists can be created with --generate-whitelist",
     )
     parser.add_argument(
         "--concise", action="store_true", help="Make output concise",
     )
     parser.add_argument(
-        "--output-whitelist",
+        "--generate-whitelist",
         action="store_true",
         help="Print a whitelist (to stdout) to be used with --whitelist",
     )
@@ -714,8 +714,8 @@ def main() -> int:
         with open(args.whitelist) as f:
             whitelist = {l.strip(): False for l in f.readlines()}
 
-    # If we need to output a whitelist, we store Error.object_desc for each error here.
-    output_whitelist = set()
+    # If we need to generate a whitelist, we store Error.object_desc for each error here.
+    generated_whitelist = set()
 
     modules = args.modules
     if args.check_typeshed:
@@ -745,8 +745,8 @@ def main() -> int:
 
             # We have errors, so change exit code, and output whatever necessary
             exit_code = 1
-            if args.output_whitelist:
-                output_whitelist.add(error.object_desc)
+            if args.generate_whitelist:
+                generated_whitelist.add(error.object_desc)
                 continue
             print(error.get_description(concise=args.concise))
 
@@ -757,8 +757,8 @@ def main() -> int:
             print(f"note: unused whitelist entry {w}")
 
     # Print the generated whitelist
-    if args.output_whitelist:
-        for e in sorted(output_whitelist):
+    if args.generate_whitelist:
+        for e in sorted(generated_whitelist):
             print(e)
         exit_code = 0
 

From 49dab438c959e14d8c5e7ddedb2ec960ab677dcc Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 21 Jan 2020 01:24:16 -0600
Subject: [PATCH 48/74] stubtest: [minor] suppress warnings

---
 scripts/stubtest.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 685b7be7ae36..01e09cbf461c 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -10,6 +10,7 @@
 import subprocess
 import sys
 import types
+import warnings
 from functools import singledispatch
 from pathlib import Path
 from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, Union
@@ -158,7 +159,10 @@ def test_module(module_name: str) -> Iterator[Error]:
         yield Error([module_name], f"failed to import: {e}", stub, MISSING)
         return
 
-    yield from verify(stub, runtime, [module_name])
+    # collections likes to warn us about the things we're doing
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        yield from verify(stub, runtime, [module_name])
 
 
 @singledispatch

From 23bd82d0b7276a726807a235494b0e9d165e8b6c Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 21 Jan 2020 19:12:59 -0600
Subject: [PATCH 49/74] stubtest: look into the mro for attributes

Prevents false positives when not using --ignore-missing-stub
---
 scripts/stubtest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 01e09cbf461c..7771454bcd6e 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -231,7 +231,7 @@ def verify_typeinfo(
 
     for entry in sorted(to_check):
         yield from verify(
-            stub.names[entry].node if entry in stub.names else MISSING,
+            next((t.names[entry].node for t in stub.mro if entry in t.names), MISSING),
             getattr(runtime, entry, MISSING),
             object_path + [entry],
         )

From 1b5a18e38b39d6864fc8d94eb5eaa0bdfe3f287a Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 21 Jan 2020 20:03:07 -0600
Subject: [PATCH 50/74] stubtest: better support @property and other decorators

---
 scripts/stubtest.py | 43 +++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 41 insertions(+), 2 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 7771454bcd6e..e1ea473a02c4 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -524,10 +524,50 @@ def verify_typevarexpr(
         yield None
 
 
+def _verify_property(stub: nodes.Decorator, runtime: Any) -> Iterator[str]:
+    assert stub.func.is_property
+    if isinstance(runtime, property):
+        return
+    if inspect.isdatadescriptor(runtime):
+        # It's enough like a property...
+        return
+    # Sometimes attributes pretend to be properties, for instance, to express that they
+    # are read only. So whitelist if runtime_type matches the return type of stub.
+    runtime_type = get_mypy_type_of_runtime_value(runtime)
+    func_type = (
+        stub.func.type.ret_type
+        if isinstance(stub.func.type, mypy.types.CallableType)
+        else None
+    )
+    if (
+        runtime_type is not None
+        and func_type is not None
+        and is_subtype_helper(runtime_type, func_type)
+    ):
+        return
+    yield "is inconsistent, cannot reconcile @property on stub with runtime object"
+
+
 @verify.register(nodes.Decorator)
 def verify_decorator(
     stub: nodes.Decorator, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
+    if isinstance(runtime, Missing):
+        yield Error(object_path, "is not present at runtime", stub, runtime)
+        return
+    if not stub.decorators:
+        # semanal.SemanticAnalyzer.visit_decorator lists the decorators that get removed (note they
+        # can still be found in stub.original_decorators).
+        if stub.func.is_property:
+            for message in _verify_property(stub, runtime):
+                yield Error(
+                    object_path, message, stub, runtime,
+                )
+            return
+
+        # For any of those decorators that aren't @property, just delegate to verify_funcitem
+        yield from verify(stub.func, runtime, object_path)
+        return
     if (
         len(stub.decorators) == 1
         and isinstance(stub.decorators[0], nodes.NameExpr)
@@ -535,8 +575,7 @@ def verify_decorator(
     ):
         # If the only decorator is @typing.overload, just delegate to the usual verify_funcitem
         yield from verify(stub.func, runtime, object_path)
-
-    # TODO: See if there are other common decorators we should be checking
+        return
 
 
 @verify.register(nodes.TypeAlias)

From 30f5f4ab47f2c1688df98886c18c2510963961a4 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 21 Jan 2020 20:19:33 -0600
Subject: [PATCH 51/74] stubtest: check classmethod and staticmethod

---
 scripts/stubtest.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index e1ea473a02c4..31e65a990528 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -254,6 +254,23 @@ def verify_funcitem(
         yield Error(object_path, "is not a function", stub, runtime)
         return
 
+    if isinstance(runtime, classmethod) and not stub.is_class:
+        yield Error(
+            object_path, "runtime is a classmethod but stub is not", stub, runtime
+        )
+    if not isinstance(runtime, classmethod) and stub.is_class:
+        yield Error(
+            object_path, "stub is a classmethod but runtime is not", stub, runtime
+        )
+    if isinstance(runtime, staticmethod) and not stub.is_static:
+        yield Error(
+            object_path, "runtime is a staticmethod but stub is not", stub, runtime
+        )
+    if not isinstance(runtime, classmethod) and stub.is_static:
+        yield Error(
+            object_path, "stub is a staticmethod but runtime is not", stub, runtime
+        )
+
     try:
         signature = inspect.signature(runtime)
     except ValueError:

From 31a299c873bcb0b7c10d3562c10610ff52e676aa Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 21 Jan 2020 21:03:46 -0600
Subject: [PATCH 52/74] stubtest: [minor] support comments in whitelist

---
 scripts/stubtest.py | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 31e65a990528..49abe6338086 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -735,6 +735,23 @@ def get_typeshed_stdlib_modules(custom_typeshed_dir: Optional[str]) -> List[str]
     return sorted(modules)
 
 
+def get_whitelist_entries(whitelist_file: Optional[str]) -> Iterator[str]:
+    if not whitelist_file:
+        return
+
+    def strip_comments(s: str) -> str:
+        try:
+            return s[: s.index("#")].strip()
+        except ValueError:
+            return s.strip()
+
+    with open(whitelist_file) as f:
+        for line in f.readlines():
+            entry = strip_comments(line)
+            if entry:
+                yield entry
+
+
 def main() -> int:
     assert sys.version_info >= (3, 6), "This script requires at least Python 3.6"
 
@@ -767,12 +784,9 @@ def main() -> int:
     )
     args = parser.parse_args()
 
-    whitelist = {}
-    if args.whitelist:
-        # Load the whitelist. This is a series of strings corresponding to Error.object_desc
-        # Values in the dict will store whether we used the whitelist entry or not.
-        with open(args.whitelist) as f:
-            whitelist = {l.strip(): False for l in f.readlines()}
+    # Load the whitelist. This is a series of strings corresponding to Error.object_desc
+    # Values in the dict will store whether we used the whitelist entry or not.
+    whitelist = {entry: False for entry in get_whitelist_entries(args.whitelist)}
 
     # If we need to generate a whitelist, we store Error.object_desc for each error here.
     generated_whitelist = set()

From 8a37fdaa2d3658bea2cfbaf90a35640e8a130955 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 21 Jan 2020 22:07:18 -0600
Subject: [PATCH 53/74] stubtest: [refactor] split up verify_funcitem

---
 scripts/stubtest.py | 405 +++++++++++++++++++++++---------------------
 1 file changed, 208 insertions(+), 197 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 49abe6338086..534e74d65cb1 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -13,7 +13,17 @@
 import warnings
 from functools import singledispatch
 from pathlib import Path
-from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Generic,
+    Iterator,
+    List,
+    Optional,
+    TypeVar,
+    Union,
+)
 
 from typing_extensions import Type
 
@@ -237,172 +247,145 @@ def verify_typeinfo(
         )
 
 
-@verify.register(nodes.FuncItem)
-def verify_funcitem(
-    stub: nodes.FuncItem,
-    runtime: MaybeMissing[types.FunctionType],
-    object_path: List[str],
-) -> Iterator[Error]:
-    if isinstance(runtime, Missing):
-        yield Error(object_path, "is not present at runtime", stub, runtime)
-        return
-    if (
-        not isinstance(runtime, (types.FunctionType, types.BuiltinFunctionType))
-        and not isinstance(runtime, (types.MethodType, types.BuiltinMethodType))
-        and not inspect.ismethoddescriptor(runtime)
-    ):
-        yield Error(object_path, "is not a function", stub, runtime)
-        return
-
+def _verify_static_class_methods(
+    stub: nodes.FuncItem, runtime: types.FunctionType
+) -> Iterator[str]:
     if isinstance(runtime, classmethod) and not stub.is_class:
-        yield Error(
-            object_path, "runtime is a classmethod but stub is not", stub, runtime
-        )
+        yield "runtime is a classmethod but stub is not"
     if not isinstance(runtime, classmethod) and stub.is_class:
-        yield Error(
-            object_path, "stub is a classmethod but runtime is not", stub, runtime
-        )
+        yield "stub is a classmethod but runtime is not"
     if isinstance(runtime, staticmethod) and not stub.is_static:
-        yield Error(
-            object_path, "runtime is a staticmethod but stub is not", stub, runtime
-        )
+        yield "runtime is a staticmethod but stub is not"
     if not isinstance(runtime, classmethod) and stub.is_static:
-        yield Error(
-            object_path, "stub is a staticmethod but runtime is not", stub, runtime
-        )
-
-    try:
-        signature = inspect.signature(runtime)
-    except ValueError:
-        # inspect.signature throws sometimes
-        return
+        yield "stub is a staticmethod but runtime is not"
 
-    def runtime_printer(s: Any) -> str:
-        return "def " + str(inspect.signature(s))
-
-    def make_error(message: str) -> Error:
-        return Error(
-            object_path,
-            "is inconsistent, " + message,
-            stub,
-            runtime,
-            runtime_printer=runtime_printer,
-        )
 
-    # Extract various arguments by type from the stub
-    stub_args_pos = []
-    stub_args_kwonly = {}
-    stub_args_varpos = None
-    stub_args_varkw = None
-
-    for stub_arg in stub.arguments:
-        if stub_arg.kind in (nodes.ARG_POS, nodes.ARG_OPT):
-            stub_args_pos.append(stub_arg)
-        elif stub_arg.kind in (nodes.ARG_NAMED, nodes.ARG_NAMED_OPT):
-            stub_args_kwonly[stub_arg.variable.name] = stub_arg
-        elif stub_arg.kind == nodes.ARG_STAR:
-            stub_args_varpos = stub_arg
-        elif stub_arg.kind == nodes.ARG_STAR2:
-            stub_args_varkw = stub_arg
-        else:
-            assert False
+def _verify_arg_name(
+    stub_arg: nodes.Argument, runtime_arg: inspect.Parameter, function_name: str
+) -> Iterator[str]:
+    """Checks whether argument names match."""
+    # Ignore exact names for all dunder methods other than __init__
+    if function_name != "__init__" and function_name.startswith("__"):
+        return
 
-    # Extract various arguments by type from the runtime object
-    runtime_args_pos = []
-    runtime_args_kwonly = {}
-    runtime_args_varpos = None
-    runtime_args_varkw = None
+    def strip_prefix(s: str, prefix: str) -> str:
+        return s[len(prefix) :] if s.startswith(prefix) else s
 
-    for runtime_arg in signature.parameters.values():
-        if runtime_arg.kind in (
-            inspect.Parameter.POSITIONAL_ONLY,
-            inspect.Parameter.POSITIONAL_OR_KEYWORD,
-        ):
-            runtime_args_pos.append(runtime_arg)
-        elif runtime_arg.kind == inspect.Parameter.KEYWORD_ONLY:
-            runtime_args_kwonly[runtime_arg.name] = runtime_arg
-        elif runtime_arg.kind == inspect.Parameter.VAR_POSITIONAL:
-            runtime_args_varpos = runtime_arg
-        elif runtime_arg.kind == inspect.Parameter.VAR_KEYWORD:
-            runtime_args_varkw = runtime_arg
-        else:
-            assert False
+    if strip_prefix(stub_arg.variable.name, "__") == runtime_arg.name:
+        return
 
-    def verify_arg_name(
-        stub_arg: nodes.Argument, runtime_arg: inspect.Parameter
-    ) -> Iterator[Error]:
-        """Checks whether argument names match."""
-        # Ignore exact names for all dunder methods other than __init__
-        if stub.name != "__init__" and stub.name.startswith("__"):
-            return
+    def names_approx_match(a: str, b: str) -> bool:
+        a = a.strip("_")
+        b = b.strip("_")
+        return a.startswith(b) or b.startswith(a) or len(a) == 1 or len(b) == 1
 
-        def strip_prefix(s: str, prefix: str) -> str:
-            return s[len(prefix) :] if s.startswith(prefix) else s
+    # Be more permissive about names matching for positional-only arguments
+    if runtime_arg.kind == inspect.Parameter.POSITIONAL_ONLY and names_approx_match(
+        stub_arg.variable.name, runtime_arg.name
+    ):
+        return
+    # This comes up with namedtuples, so ignore
+    if stub_arg.variable.name == "_self":
+        return
+    yield (
+        f'stub argument "{stub_arg.variable.name}" differs from '
+        f'runtime argument "{runtime_arg.name}"'
+    )
 
-        if strip_prefix(stub_arg.variable.name, "__") == runtime_arg.name:
-            return
 
-        def names_approx_match(a: str, b: str) -> bool:
-            a = a.strip("_")
-            b = b.strip("_")
-            return a.startswith(b) or b.startswith(a) or len(a) == 1 or len(b) == 1
+def _verify_arg_default_value(
+    stub_arg: nodes.Argument, runtime_arg: inspect.Parameter
+) -> Iterator[str]:
+    """Checks whether argument default values are compatible."""
+    if runtime_arg.default != inspect.Parameter.empty:
+        if stub_arg.kind not in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
+            yield (
+                f'runtime argument "{runtime_arg.name}" has a default value '
+                "but stub argument does not"
+            )
+        else:
+            runtime_type = get_mypy_type_of_runtime_value(runtime_arg.default)
+            if (
+                runtime_type is not None
+                and stub_arg.variable.type is not None
+                # Avoid false positives for marker objects
+                and type(runtime_arg.default) != object
+                and not is_subtype_helper(runtime_type, stub_arg.variable.type)
+            ):
+                yield (
+                    f'runtime argument "{runtime_arg.name}" has a default value of type '
+                    f"{runtime_type}, which is incompatible with stub argument type "
+                    f"{stub_arg.variable.type}"
+                )
+    else:
+        if stub_arg.kind in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
+            yield (
+                f'stub argument "{stub_arg.variable.name}" has a default '
+                "value but runtime argument does not"
+            )
 
-        # Be more permissive about names matching for positional-only arguments
-        if (
-            runtime_arg.kind == inspect.Parameter.POSITIONAL_ONLY
-            and names_approx_match(stub_arg.variable.name, runtime_arg.name)
-        ):
-            return
-        # This comes up with namedtuples, so ignore
-        if stub_arg.variable.name == "_self":
-            return
-        yield make_error(
-            f'stub argument "{stub_arg.variable.name}" differs from '
-            f'runtime argument "{runtime_arg.name}"'
-        )
 
-    def verify_arg_default_value(
-        stub_arg: nodes.Argument, runtime_arg: inspect.Parameter
-    ) -> Iterator[Error]:
-        """Checks whether argument default values are compatible."""
-        if runtime_arg.default != inspect.Parameter.empty:
-            if stub_arg.kind not in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
-                yield make_error(
-                    f'runtime argument "{runtime_arg.name}" has a default value '
-                    "but stub argument does not"
-                )
+class Signature(Generic[T]):
+    def __init__(self) -> None:
+        self.pos: List[T] = []
+        self.kwonly: Dict[str, T] = {}
+        self.varpos: Optional[T] = None
+        self.varkw: Optional[T] = None
+
+    @staticmethod
+    def from_funcitem(stub: nodes.FuncItem) -> "Signature[nodes.Argument]":
+        stub_sig: Signature[nodes.Argument] = Signature()
+        for stub_arg in stub.arguments:
+            if stub_arg.kind in (nodes.ARG_POS, nodes.ARG_OPT):
+                stub_sig.pos.append(stub_arg)
+            elif stub_arg.kind in (nodes.ARG_NAMED, nodes.ARG_NAMED_OPT):
+                stub_sig.kwonly[stub_arg.variable.name] = stub_arg
+            elif stub_arg.kind == nodes.ARG_STAR:
+                stub_sig.varpos = stub_arg
+            elif stub_arg.kind == nodes.ARG_STAR2:
+                stub_sig.varkw = stub_arg
             else:
-                runtime_type = get_mypy_type_of_runtime_value(runtime_arg.default)
-                if (
-                    runtime_type is not None
-                    and stub_arg.variable.type is not None
-                    # Avoid false positives for marker objects
-                    and type(runtime_arg.default) != object
-                    and not is_subtype_helper(runtime_type, stub_arg.variable.type)
-                ):
-                    yield make_error(
-                        f'runtime argument "{runtime_arg.name}" has a default value of type '
-                        f"{runtime_type}, which is incompatible with stub argument type "
-                        f"{stub_arg.variable.type}"
-                    )
-        else:
-            if stub_arg.kind in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
-                yield make_error(
-                    f'stub argument "{stub_arg.variable.name}" has a default '
-                    "value but runtime argument does not"
-                )
+                raise ValueError
+        return stub_sig
+
+    @staticmethod
+    def from_inspect_signature(
+        signature: inspect.Signature,
+    ) -> "Signature[inspect.Parameter]":
+        runtime_sig: Signature[inspect.Parameter] = Signature()
+        for runtime_arg in signature.parameters.values():
+            if runtime_arg.kind in (
+                inspect.Parameter.POSITIONAL_ONLY,
+                inspect.Parameter.POSITIONAL_OR_KEYWORD,
+            ):
+                runtime_sig.pos.append(runtime_arg)
+            elif runtime_arg.kind == inspect.Parameter.KEYWORD_ONLY:
+                runtime_sig.kwonly[runtime_arg.name] = runtime_arg
+            elif runtime_arg.kind == inspect.Parameter.VAR_POSITIONAL:
+                runtime_sig.varpos = runtime_arg
+            elif runtime_arg.kind == inspect.Parameter.VAR_KEYWORD:
+                runtime_sig.varkw = runtime_arg
+            else:
+                raise ValueError
+        return runtime_sig
+
 
+def _verify_signature(
+    stub: Signature[nodes.Argument],
+    runtime: Signature[inspect.Parameter],
+    function_name: str,
+) -> Iterator[str]:
     # Check positional arguments match up
-    for stub_arg, runtime_arg in zip(stub_args_pos, runtime_args_pos):
-        yield from verify_arg_name(stub_arg, runtime_arg)
-        yield from verify_arg_default_value(stub_arg, runtime_arg)
+    for stub_arg, runtime_arg in zip(stub.pos, runtime.pos):
+        yield from _verify_arg_name(stub_arg, runtime_arg, function_name)
+        yield from _verify_arg_default_value(stub_arg, runtime_arg)
         if (
             runtime_arg.kind == inspect.Parameter.POSITIONAL_ONLY
             and not stub_arg.variable.name.startswith("__")
             and not stub_arg.variable.name.strip("_") == "self"
-            and not stub.name.startswith("__")  # noisy for dunder methods
+            and not function_name.startswith("__")  # noisy for dunder methods
         ):
-            yield make_error(
+            yield (
                 f'stub argument "{stub_arg.variable.name}" should be '
                 "positional-only (rename with a leading double underscore)"
             )
@@ -410,77 +393,105 @@ def verify_arg_default_value(
             runtime_arg.kind != inspect.Parameter.POSITIONAL_ONLY
             and stub_arg.variable.name.startswith("__")
         ):
-            yield make_error(
+            yield (
                 f'stub argument "{stub_arg.variable.name}" is positional or keyword '
                 "(remove leading double underscore)"
             )
 
     # Checks involving *args
-    if len(stub_args_pos) == len(runtime_args_pos):
-        if stub_args_varpos is None and runtime_args_varpos is not None:
-            yield make_error(
-                f'stub does not have *args argument "{runtime_args_varpos.name}"'
-            )
-        if stub_args_varpos is not None and runtime_args_varpos is None:
-            yield make_error(
-                f'runtime does not have *args argument "{stub_args_varpos.variable.name}"'
+    if len(stub.pos) == len(runtime.pos):
+        if stub.varpos is None and runtime.varpos is not None:
+            yield f'stub does not have *args argument "{runtime.varpos.name}"'
+        if stub.varpos is not None and runtime.varpos is None:
+            yield (
+                f'runtime does not have *args argument "{stub.varpos.variable.name}"'
             )
-    elif len(stub_args_pos) > len(runtime_args_pos):
-        if runtime_args_varpos is None:
-            for stub_arg in stub_args_pos[len(runtime_args_pos) :]:
-                # If the variable is in runtime_args_kwonly, it's just mislabelled as not a
+    elif len(stub.pos) > len(runtime.pos):
+        if runtime.varpos is None:
+            for stub_arg in stub.pos[len(runtime.pos) :]:
+                # If the variable is in runtime.kwonly, it's just mislabelled as not a
                 # keyword-only argument; we report the error while checking keyword-only arguments
-                if stub_arg.variable.name not in runtime_args_kwonly:
-                    yield make_error(
-                        f'runtime does not have argument "{stub_arg.variable.name}"'
-                    )
+                if stub_arg.variable.name not in runtime.kwonly:
+                    yield f'runtime does not have argument "{stub_arg.variable.name}"'
         # We do not check whether stub takes *args when the runtime does, for cases where the stub
         # just listed out the extra parameters the function takes
-    elif len(stub_args_pos) < len(runtime_args_pos):
-        if stub_args_varpos is None:
-            for runtime_arg in runtime_args_pos[len(stub_args_pos) :]:
-                yield make_error(f'stub does not have argument "{runtime_arg.name}"')
-        elif runtime_args_pos is None:
-            yield make_error(
-                f'runtime does not have *args argument "{stub_args_varpos.variable.name}"'
+    elif len(stub.pos) < len(runtime.pos):
+        if stub.varpos is None:
+            for runtime_arg in runtime.pos[len(stub.pos) :]:
+                yield f'stub does not have argument "{runtime_arg.name}"'
+        elif runtime.pos is None:
+            yield (
+                f'runtime does not have *args argument "{stub.varpos.variable.name}"'
             )
 
     # Check keyword-only args
-    for arg in sorted(set(stub_args_kwonly) & set(runtime_args_kwonly)):
-        stub_arg, runtime_arg = stub_args_kwonly[arg], runtime_args_kwonly[arg]
-        yield from verify_arg_name(stub_arg, runtime_arg)
-        yield from verify_arg_default_value(stub_arg, runtime_arg)
+    for arg in sorted(set(stub.kwonly) & set(runtime.kwonly)):
+        stub_arg, runtime_arg = stub.kwonly[arg], runtime.kwonly[arg]
+        yield from _verify_arg_name(stub_arg, runtime_arg, function_name)
+        yield from _verify_arg_default_value(stub_arg, runtime_arg)
 
     # Checks involving **kwargs
-    if stub_args_varkw is None and runtime_args_varkw is not None:
+    if stub.varkw is None and runtime.varkw is not None:
         # We do not check whether stub takes **kwargs when the runtime does, for cases where the
         # stub just listed out the extra keyword parameters the function takes
         # Also check against positional parameters, to avoid a nitpicky message when an argument
         # isn't marked as keyword-only
-        stub_pos_names = set(stub_arg.variable.name for stub_arg in stub_args_pos)
-        if not set(runtime_args_kwonly).issubset(
-            set(stub_args_kwonly) | stub_pos_names
-        ):
-            yield make_error(
-                f'stub does not have **kwargs argument "{runtime_args_varkw.name}"'
-            )
-    if stub_args_varkw is not None and runtime_args_varkw is None:
-        yield make_error(
-            f'runtime does not have **kwargs argument "{stub_args_varkw.variable.name}"'
-        )
-    if runtime_args_varkw is None or not set(runtime_args_kwonly).issubset(
-        set(stub_args_kwonly)
-    ):
-        for arg in sorted(set(stub_args_kwonly) - set(runtime_args_kwonly)):
-            yield make_error(f'runtime does not have argument "{arg}"')
-    if stub_args_varkw is None or not set(stub_args_kwonly).issubset(
-        set(runtime_args_kwonly)
-    ):
-        for arg in sorted(set(runtime_args_kwonly) - set(stub_args_kwonly)):
-            if arg in set(stub_arg.variable.name for stub_arg in stub_args_pos):
-                yield make_error(f'stub argument "{arg}" is not keyword-only')
+        stub_pos_names = set(stub_arg.variable.name for stub_arg in stub.pos)
+        if not set(runtime.kwonly).issubset(set(stub.kwonly) | stub_pos_names):
+            yield f'stub does not have **kwargs argument "{runtime.varkw.name}"'
+    if stub.varkw is not None and runtime.varkw is None:
+        yield f'runtime does not have **kwargs argument "{stub.varkw.variable.name}"'
+    if runtime.varkw is None or not set(runtime.kwonly).issubset(set(stub.kwonly)):
+        for arg in sorted(set(stub.kwonly) - set(runtime.kwonly)):
+            yield f'runtime does not have argument "{arg}"'
+    if stub.varkw is None or not set(stub.kwonly).issubset(set(runtime.kwonly)):
+        for arg in sorted(set(runtime.kwonly) - set(stub.kwonly)):
+            if arg in set(stub_arg.variable.name for stub_arg in stub.pos):
+                yield f'stub argument "{arg}" is not keyword-only'
             else:
-                yield make_error(f'stub does not have argument "{arg}"')
+                yield f'stub does not have argument "{arg}"'
+
+
+@verify.register(nodes.FuncItem)
+def verify_funcitem(
+    stub: nodes.FuncItem,
+    runtime: MaybeMissing[types.FunctionType],
+    object_path: List[str],
+) -> Iterator[Error]:
+    if isinstance(runtime, Missing):
+        yield Error(object_path, "is not present at runtime", stub, runtime)
+        return
+    if (
+        not isinstance(runtime, (types.FunctionType, types.BuiltinFunctionType))
+        and not isinstance(runtime, (types.MethodType, types.BuiltinMethodType))
+        and not inspect.ismethoddescriptor(runtime)
+    ):
+        yield Error(object_path, "is not a function", stub, runtime)
+        return
+
+    for message in _verify_static_class_methods(stub, runtime):
+        yield Error(object_path, message, stub, runtime)
+
+    try:
+        signature = inspect.signature(runtime)
+    except ValueError:
+        # inspect.signature throws sometimes
+        return
+
+    stub_sig = Signature.from_funcitem(stub)
+    runtime_sig = Signature.from_inspect_signature(signature)
+
+    def runtime_printer(s: Any) -> str:
+        return "def " + str(inspect.signature(s))
+
+    for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name):
+        yield Error(
+            object_path,
+            "is inconsistent, " + message,
+            stub,
+            runtime,
+            runtime_printer=runtime_printer,
+        )
 
 
 @verify.register(Missing)

From da90df733ef28485d758a9dd0f864d1f2814bbde Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 22 Jan 2020 00:49:45 -0600
Subject: [PATCH 54/74] stubtest: [minor] suggest positional-only name

The name doesn't matter, but if someone's fixing something, we might as
well make it easy to match runtime name
---
 scripts/stubtest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 534e74d65cb1..263d63fcb440 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -386,8 +386,8 @@ def _verify_signature(
             and not function_name.startswith("__")  # noisy for dunder methods
         ):
             yield (
-                f'stub argument "{stub_arg.variable.name}" should be '
-                "positional-only (rename with a leading double underscore)"
+                f'stub argument "{stub_arg.variable.name}" should be positional-only '
+                f'(rename with a leading double underscore, i.e. "__{runtime_arg.name}")'
             )
         if (
             runtime_arg.kind != inspect.Parameter.POSITIONAL_ONLY

From 4c05bca5152ab1fc3ef47db182ef6ceeb22c9637 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 22 Jan 2020 22:14:13 -0800
Subject: [PATCH 55/74] stubtest: add __str__ for Signature

---
 scripts/stubtest.py | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 263d63fcb440..07557c6fd8d3 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -332,6 +332,50 @@ def __init__(self) -> None:
         self.varpos: Optional[T] = None
         self.varkw: Optional[T] = None
 
+    def __str__(self) -> str:
+        def get_name(arg: Any) -> str:
+            if isinstance(arg, inspect.Parameter):
+                return arg.name
+            if isinstance(arg, nodes.Argument):
+                return arg.variable.name
+            raise ValueError
+
+        def get_type(arg: Any) -> Optional[str]:
+            if isinstance(arg, inspect.Parameter):
+                return None
+            if isinstance(arg, nodes.Argument):
+                return str(arg.variable.type or arg.type_annotation)
+            raise ValueError
+
+        def has_default(arg: Any) -> bool:
+            if isinstance(arg, inspect.Parameter):
+                return arg.default != inspect.Parameter.empty
+            if isinstance(arg, nodes.Argument):
+                return arg.kind in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT)
+            raise ValueError
+
+        def get_desc(arg: Any) -> str:
+            arg_type = get_type(arg)
+            return (
+                get_name(arg)
+                + (f": {arg_type}" if arg_type else "")
+                + (" = ..." if has_default(arg) else "")
+            )
+
+        ret = "def ("
+        ret += ", ".join(
+            [get_desc(arg) for arg in self.pos]
+            + (
+                ["*" + get_name(self.varpos)]
+                if self.varpos
+                else (["*"] if self.kwonly else [])
+            )
+            + [get_desc(arg) for arg in self.kwonly.values()]
+            + (["**" + get_name(self.varkw)] if self.varkw else [])
+        )
+        ret += ")"
+        return ret
+
     @staticmethod
     def from_funcitem(stub: nodes.FuncItem) -> "Signature[nodes.Argument]":
         stub_sig: Signature[nodes.Argument] = Signature()

From 3afda913e34bc3ded199d178db6a7e988cb4cf61 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 22 Jan 2020 22:18:11 -0800
Subject: [PATCH 56/74] stubtest: implement smarter overload checking

This eliminates ~400 false positives
---
 scripts/stubtest.py | 152 +++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 136 insertions(+), 16 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 07557c6fd8d3..0dd39cc033b6 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -21,6 +21,7 @@
     Iterator,
     List,
     Optional,
+    Tuple,
     TypeVar,
     Union,
 )
@@ -305,17 +306,21 @@ def _verify_arg_default_value(
             )
         else:
             runtime_type = get_mypy_type_of_runtime_value(runtime_arg.default)
+            # Fallback to the type annotation type if var type is missing. The type annotation
+            # is an UnboundType, but I don't know enough to know what the pros and cons here are.
+            # UnboundTypes have ugly question marks following them, so default to var type.
+            # Note we do this same fallback when constructing signatures in from_overloadedfuncdef
+            stub_type = stub_arg.variable.type or stub_arg.type_annotation
             if (
                 runtime_type is not None
-                and stub_arg.variable.type is not None
+                and stub_type is not None
                 # Avoid false positives for marker objects
                 and type(runtime_arg.default) != object
-                and not is_subtype_helper(runtime_type, stub_arg.variable.type)
+                and not is_subtype_helper(runtime_type, stub_type)
             ):
                 yield (
                     f'runtime argument "{runtime_arg.name}" has a default value of type '
-                    f"{runtime_type}, which is incompatible with stub argument type "
-                    f"{stub_arg.variable.type}"
+                    f"{runtime_type}, which is incompatible with stub argument type {stub_type}"
                 )
     else:
         if stub_arg.kind in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
@@ -413,6 +418,99 @@ def from_inspect_signature(
                 raise ValueError
         return runtime_sig
 
+    @staticmethod
+    def from_overloadedfuncdef(
+        stub: nodes.OverloadedFuncDef,
+    ) -> "Signature[nodes.Argument]":
+        """Returns a Signature from an OverloadedFuncDef.
+
+        If life were simple, to verify_overloadedfuncdef, we'd just verify_funcitem for each of its
+        items. Unfortunately, life isn't simple and overloads are pretty deceitful. So instead, we
+        try and combine the overload's items into a single signature that is compatible with any
+        lies it might try to tell.
+
+        """
+
+        def get_funcs() -> Iterator[nodes.FuncItem]:
+            for dec in stub.items:
+                if (
+                    isinstance(dec, nodes.Decorator)
+                    and len(dec.decorators) == 1
+                    and isinstance(dec.decorators[0], nodes.NameExpr)
+                    and dec.decorators[0].fullname == "typing.overload"
+                ):
+                    yield dec.func
+                else:
+                    raise ValueError
+
+        # For all dunder methods other than __init__, just assume all args are positional-only
+        assume_positional_only = stub.name != "__init__" and stub.name.startswith("__")
+
+        all_args: Dict[str, List[Tuple[nodes.Argument, int]]] = {}
+        for func in get_funcs():
+            for index, arg in enumerate(func.arguments):
+                # For positional-only args, we allow overloads to have different names for the same
+                # argument. To accomplish this, we just make up a fake index-based name.
+                name = (
+                    f"__{index}"
+                    if arg.variable.name.startswith("__") or assume_positional_only
+                    else arg.variable.name
+                )
+                all_args.setdefault(name, []).append((arg, index))
+
+        def get_position(arg_name: str) -> int:
+            # We just need this to return the positional args in the correct order.
+            return max(index for _, index in all_args[arg_name])
+
+        def get_type(arg_name: str) -> mypy.types.ProperType:
+            with mypy.state.strict_optional_set(True):
+                all_types = [
+                    arg.variable.type or arg.type_annotation
+                    for arg, _ in all_args[arg_name]
+                ]
+                return mypy.typeops.make_simplified_union([t for t in all_types if t])
+
+        def get_kind(arg_name: str) -> int:
+            kinds = {arg.kind for arg, _ in all_args[arg_name]}
+            if nodes.ARG_STAR in kinds:
+                return nodes.ARG_STAR
+            if nodes.ARG_STAR2 in kinds:
+                return nodes.ARG_STAR2
+            # The logic here is based on two tenets:
+            # 1) If an arg is ever optional (or unspecified), it is optional
+            # 2) If an arg is ever positional, it is positional
+            is_opt = (
+                len(all_args[arg_name]) < len(stub.items)
+                or nodes.ARG_OPT in kinds
+                or nodes.ARG_NAMED_OPT in kinds
+            )
+            is_pos = nodes.ARG_OPT in kinds or nodes.ARG_POS in kinds
+            if is_opt:
+                return nodes.ARG_OPT if is_pos else nodes.ARG_NAMED_OPT
+            return nodes.ARG_POS if is_pos else nodes.ARG_NAMED
+
+        sig: Signature[nodes.Argument] = Signature()
+        for arg_name in sorted(all_args, key=get_position):
+            # example_arg_name gives us a real name (in case we had a fake index-based name)
+            example_arg_name = all_args[arg_name][0][0].variable.name
+            arg = nodes.Argument(
+                nodes.Var(example_arg_name, get_type(arg_name)),
+                type_annotation=None,
+                initializer=None,
+                kind=get_kind(arg_name),
+            )
+            if arg.kind in (nodes.ARG_POS, nodes.ARG_OPT):
+                sig.pos.append(arg)
+            elif arg.kind in (nodes.ARG_NAMED, nodes.ARG_NAMED_OPT):
+                sig.kwonly[arg.variable.name] = arg
+            elif arg.kind == nodes.ARG_STAR:
+                sig.varpos = arg
+            elif arg.kind == nodes.ARG_STAR2:
+                sig.varkw = arg
+            else:
+                raise ValueError
+        return sig
+
 
 def _verify_signature(
     stub: Signature[nodes.Argument],
@@ -582,10 +680,37 @@ def verify_var(
 def verify_overloadedfuncdef(
     stub: nodes.OverloadedFuncDef, runtime: MaybeMissing[Any], object_path: List[str]
 ) -> Iterator[Error]:
-    # TODO: Overloads can be pretty deceitful, so maybe be more permissive when dealing with them
-    # For a motivating example, look at RawConfigParser.items and RawConfigParser.get
-    for func in stub.items:
-        yield from verify(func, runtime, object_path)
+    if isinstance(runtime, Missing):
+        yield Error(object_path, "is not present at runtime", stub, runtime)
+        return
+
+    try:
+        stub_sig = Signature.from_overloadedfuncdef(stub)
+        runtime_sig = Signature.from_inspect_signature(inspect.signature(runtime))
+    except ValueError:
+        return
+
+    def stub_printer(s: Any) -> str:
+        return str(s.type) + "\nInferred signature: " + str(stub_sig)
+
+    def runtime_printer(s: Any) -> str:
+        return "def " + str(inspect.signature(s))
+
+    for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name):
+        # TODO: This is a little hacky, but the addition here is super useful
+        if "has a default value of type" in message:
+            message += (
+                ". This is often caused by overloads failing to account for explicitly passing "
+                "in the default value."
+            )
+        yield Error(
+            object_path,
+            "is inconsistent, " + message,
+            stub,
+            runtime,
+            stub_printer=stub_printer,
+            runtime_printer=runtime_printer,
+        )
 
 
 @verify.register(nodes.TypeVarExpr)
@@ -640,14 +765,9 @@ def verify_decorator(
         # For any of those decorators that aren't @property, just delegate to verify_funcitem
         yield from verify(stub.func, runtime, object_path)
         return
-    if (
-        len(stub.decorators) == 1
-        and isinstance(stub.decorators[0], nodes.NameExpr)
-        and stub.decorators[0].fullname == "typing.overload"
-    ):
-        # If the only decorator is @typing.overload, just delegate to the usual verify_funcitem
-        yield from verify(stub.func, runtime, object_path)
-        return
+
+    # Just skip checking any other decorators. The only other relevant one when checking stdlib
+    # is @overload, which gets handled by verify_overloadedfuncdef and never reaches here.
 
 
 @verify.register(nodes.TypeAlias)

From bb82c18e7cdf03226288588713e60d1b2def3b56 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 22 Jan 2020 22:58:44 -0800
Subject: [PATCH 57/74] stubtest: improve typeinfo output, simplify
 descriptions

---
 scripts/stubtest.py | 54 ++++++++++++++++++---------------------------
 1 file changed, 22 insertions(+), 32 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 0dd39cc033b6..65e67c2991d6 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -15,7 +15,6 @@
 from pathlib import Path
 from typing import (
     Any,
-    Callable,
     Dict,
     Generic,
     Iterator,
@@ -64,8 +63,9 @@ def __init__(
         message: str,
         stub_object: MaybeMissing[nodes.Node],
         runtime_object: MaybeMissing[Any],
-        stub_printer: Optional[Callable[[nodes.Node], str]] = None,
-        runtime_printer: Optional[Callable[[Any], str]] = None,
+        *,
+        stub_desc: Optional[str] = None,
+        runtime_desc: Optional[str] = None,
     ) -> None:
         """Represents an error found by stubtest.
 
@@ -73,22 +73,16 @@ def __init__(
         :param message: Error message
         :param stub_object: The mypy node representing the stub
         :param runtime_object: Actual object obtained from the runtime
-        :param stub_printer: Callable to provide specialised output for a given stub object
-        :param runtime_printer: Callable to provide specialised output for a given runtime object
+        :param stub_desc: Specialised description for the stub object, should you wish
+        :param runtime_desc: Specialised description for the runtime object, should you wish
 
         """
         self.object_desc = ".".join(object_path)
         self.message = message
         self.stub_object = stub_object
         self.runtime_object = runtime_object
-        if stub_printer is None:
-            stub_printer = lambda stub: str(getattr(stub, "type", stub))
-        self.stub_printer = lambda s: s if isinstance(s, Missing) else stub_printer(s)
-        if runtime_printer is None:
-            runtime_printer = lambda runtime: str(runtime)
-        self.runtime_printer = (
-            lambda s: s if isinstance(s, Missing) else runtime_printer(s)
-        )
+        self.stub_desc = stub_desc or str(getattr(stub_object, "type", stub_object))
+        self.runtime_desc = runtime_desc or str(runtime_object)
 
     def is_missing_stub(self) -> bool:
         """Whether or not the error is for something missing from the stub."""
@@ -140,13 +134,11 @@ def get_description(self, concise: bool = False) -> str:
             "Stub:",
             _style(stub_loc_str, dim=True),
             "\n",
-            _style(f"{self.stub_printer(self.stub_object)}\n", color="blue", dim=True),
+            _style(self.stub_desc + "\n", color="blue", dim=True),
             "Runtime:",
             _style(runtime_loc_str, dim=True),
             "\n",
-            _style(
-                f"{self.runtime_printer(self.runtime_object)}\n", color="blue", dim=True
-            ),
+            _style(self.runtime_desc + "\n", color="blue", dim=True),
         ]
         return "".join(output)
 
@@ -231,10 +223,16 @@ def verify_typeinfo(
     stub: nodes.TypeInfo, runtime: MaybeMissing[Type[Any]], object_path: List[str]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
-        yield Error(object_path, "is not present at runtime", stub, runtime)
+        yield Error(
+            object_path,
+            "is not present at runtime",
+            stub,
+            runtime,
+            stub_desc=repr(stub),
+        )
         return
     if not isinstance(runtime, type):
-        yield Error(object_path, "is not a type", stub, runtime)
+        yield Error(object_path, "is not a type", stub, runtime, stub_desc=repr(stub))
         return
 
     to_check = set(stub.names)
@@ -623,16 +621,13 @@ def verify_funcitem(
     stub_sig = Signature.from_funcitem(stub)
     runtime_sig = Signature.from_inspect_signature(signature)
 
-    def runtime_printer(s: Any) -> str:
-        return "def " + str(inspect.signature(s))
-
     for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name):
         yield Error(
             object_path,
             "is inconsistent, " + message,
             stub,
             runtime,
-            runtime_printer=runtime_printer,
+            runtime_desc="def " + str(signature),
         )
 
 
@@ -686,16 +681,11 @@ def verify_overloadedfuncdef(
 
     try:
         stub_sig = Signature.from_overloadedfuncdef(stub)
-        runtime_sig = Signature.from_inspect_signature(inspect.signature(runtime))
+        signature = inspect.signature(runtime)
+        runtime_sig = Signature.from_inspect_signature(signature)
     except ValueError:
         return
 
-    def stub_printer(s: Any) -> str:
-        return str(s.type) + "\nInferred signature: " + str(stub_sig)
-
-    def runtime_printer(s: Any) -> str:
-        return "def " + str(inspect.signature(s))
-
     for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name):
         # TODO: This is a little hacky, but the addition here is super useful
         if "has a default value of type" in message:
@@ -708,8 +698,8 @@ def runtime_printer(s: Any) -> str:
             "is inconsistent, " + message,
             stub,
             runtime,
-            stub_printer=stub_printer,
-            runtime_printer=runtime_printer,
+            stub_desc=str(stub.type) + f"\nInferred signature: {stub_sig}",
+            runtime_desc="def " + str(signature),
         )
 
 

From 9ea10bd4a427b2af7f328d30b42ab7811a5993c3 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 24 Jan 2020 12:41:54 -0800
Subject: [PATCH 58/74] stubtest: [minor] blacken

Switched laptops, think I ran into a black version difference
---
 scripts/stubtest.py | 24 +++++-------------------
 1 file changed, 5 insertions(+), 19 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 65e67c2991d6..bbf0e98aa67e 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -13,17 +13,7 @@
 import warnings
 from functools import singledispatch
 from pathlib import Path
-from typing import (
-    Any,
-    Dict,
-    Generic,
-    Iterator,
-    List,
-    Optional,
-    Tuple,
-    TypeVar,
-    Union,
-)
+from typing import Any, Dict, Generic, Iterator, List, Optional, Tuple, TypeVar, Union
 
 from typing_extensions import Type
 
@@ -69,7 +59,8 @@ def __init__(
     ) -> None:
         """Represents an error found by stubtest.
 
-        :param object_path: Location of the object with the error, eg, ["module", "Class", "method"]
+        :param object_path: Location of the object with the error,
+            e.g. ``["module", "Class", "method"]``
         :param message: Error message
         :param stub_object: The mypy node representing the stub
         :param runtime_object: Actual object obtained from the runtime
@@ -852,10 +843,7 @@ def build_stubs(modules: List[str], options: Options) -> None:
 
     res = mypy.build.build(sources=sources, options=options)
     if res.errors:
-        output = [
-            _style("error: ", color="red", bold=True),
-            " failed mypy build.\n",
-        ]
+        output = [_style("error: ", color="red", bold=True), " failed mypy build.\n"]
         print("".join(output) + "\n".join(res.errors))
         sys.exit(1)
 
@@ -939,9 +927,7 @@ def main() -> int:
         "--whitelist",
         help="Use file as a whitelist. Whitelists can be created with --generate-whitelist",
     )
-    parser.add_argument(
-        "--concise", action="store_true", help="Make output concise",
-    )
+    parser.add_argument("--concise", action="store_true", help="Make output concise")
     parser.add_argument(
         "--generate-whitelist",
         action="store_true",

From 8686c33883ac7bc2c7cd21e21326926c577a733e Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 25 Jan 2020 17:56:23 -0800
Subject: [PATCH 59/74] stubtest: improve decorator handling, fix classmethod
 signature

---
 scripts/stubtest.py | 95 ++++++++++++++++++++++++++++++---------------
 1 file changed, 64 insertions(+), 31 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index bbf0e98aa67e..fc2eebede3b2 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -5,6 +5,7 @@
 """
 
 import argparse
+import copy
 import importlib
 import inspect
 import subprocess
@@ -419,24 +420,12 @@ def from_overloadedfuncdef(
         lies it might try to tell.
 
         """
-
-        def get_funcs() -> Iterator[nodes.FuncItem]:
-            for dec in stub.items:
-                if (
-                    isinstance(dec, nodes.Decorator)
-                    and len(dec.decorators) == 1
-                    and isinstance(dec.decorators[0], nodes.NameExpr)
-                    and dec.decorators[0].fullname == "typing.overload"
-                ):
-                    yield dec.func
-                else:
-                    raise ValueError
-
         # For all dunder methods other than __init__, just assume all args are positional-only
         assume_positional_only = stub.name != "__init__" and stub.name.startswith("__")
 
         all_args: Dict[str, List[Tuple[nodes.Argument, int]]] = {}
-        for func in get_funcs():
+        for func in map(_resolve_funcitem_from_decorator, stub.items):
+            assert func is not None
             for index, arg in enumerate(func.arguments):
                 # For positional-only args, we allow overloads to have different names for the same
                 # argument. To accomplish this, we just make up a fake index-based name.
@@ -600,7 +589,7 @@ def verify_funcitem(
         yield Error(object_path, "is not a function", stub, runtime)
         return
 
-    for message in _verify_static_class_methods(stub, runtime):
+    for message in _verify_static_class_methods(stub, runtime, object_path):
         yield Error(object_path, message, stub, runtime)
 
     try:
@@ -670,13 +659,18 @@ def verify_overloadedfuncdef(
         yield Error(object_path, "is not present at runtime", stub, runtime)
         return
 
+    if stub.is_property:
+        # We get here in cases of overloads from property.setter
+        return
+
     try:
-        stub_sig = Signature.from_overloadedfuncdef(stub)
         signature = inspect.signature(runtime)
-        runtime_sig = Signature.from_inspect_signature(signature)
     except ValueError:
         return
 
+    stub_sig = Signature.from_overloadedfuncdef(stub)
+    runtime_sig = Signature.from_inspect_signature(signature)
+
     for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name):
         # TODO: This is a little hacky, but the addition here is super useful
         if "has a default value of type" in message:
@@ -726,6 +720,53 @@ def _verify_property(stub: nodes.Decorator, runtime: Any) -> Iterator[str]:
     yield "is inconsistent, cannot reconcile @property on stub with runtime object"
 
 
+def _resolve_funcitem_from_decorator(
+    dec: nodes.OverloadPart
+) -> Optional[nodes.FuncItem]:
+    """Returns a FuncItem that corresponds to the output of the decorator.
+
+    Returns None if we can't figure out what that would be. For convenience, this function also
+    accepts FuncItems.
+
+    """
+    if isinstance(dec, nodes.FuncItem):
+        return dec
+    if dec.func.is_property:
+        return None
+
+    def apply_decorator_to_funcitem(
+        decorator: nodes.Expression, func: nodes.FuncItem
+    ) -> Optional[nodes.FuncItem]:
+        if not isinstance(decorator, nodes.NameExpr):
+            return None
+        if decorator.fullname is None:
+            # Happens with namedtuple
+            return None
+        if decorator.fullname in (
+            "builtins.staticmethod",
+            "typing.overload",
+            "abc.abstractmethod",
+        ):
+            return func
+        if decorator.fullname == "builtins.classmethod":
+            assert func.arguments[0].variable.name in ("cls", "metacls")
+            ret = copy.copy(func)
+            # Remove the cls argument, since it's not present in inspect.signature of classmethods
+            ret.arguments = ret.arguments[1:]
+            return ret
+        # Just give up on any other decorators. After excluding properties, we don't run into
+        # anything else when running on typeshed's stdlib.
+        return None
+
+    func: nodes.FuncItem = dec.func
+    for decorator in dec.original_decorators:
+        resulting_func = apply_decorator_to_funcitem(decorator, func)
+        if resulting_func is None:
+            return None
+        func = resulting_func
+    return func
+
+
 @verify.register(nodes.Decorator)
 def verify_decorator(
     stub: nodes.Decorator, runtime: MaybeMissing[Any], object_path: List[str]
@@ -733,22 +774,14 @@ def verify_decorator(
     if isinstance(runtime, Missing):
         yield Error(object_path, "is not present at runtime", stub, runtime)
         return
-    if not stub.decorators:
-        # semanal.SemanticAnalyzer.visit_decorator lists the decorators that get removed (note they
-        # can still be found in stub.original_decorators).
-        if stub.func.is_property:
-            for message in _verify_property(stub, runtime):
-                yield Error(
-                    object_path, message, stub, runtime,
-                )
-            return
-
-        # For any of those decorators that aren't @property, just delegate to verify_funcitem
-        yield from verify(stub.func, runtime, object_path)
+    if stub.func.is_property:
+        for message in _verify_property(stub, runtime):
+            yield Error(object_path, message, stub, runtime)
         return
 
-    # Just skip checking any other decorators. The only other relevant one when checking stdlib
-    # is @overload, which gets handled by verify_overloadedfuncdef and never reaches here.
+    func = _resolve_funcitem_from_decorator(stub)
+    if func is not None:
+        yield from verify(func, runtime, object_path)
 
 
 @verify.register(nodes.TypeAlias)

From f37e5881231515d6ba2448522efaf3db33576830 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 25 Jan 2020 17:57:07 -0800
Subject: [PATCH 60/74] stubtest: fix classmethod and staticmethod
 introspection

This was just broken previously
---
 scripts/stubtest.py | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index fc2eebede3b2..9bfee912a628 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -239,15 +239,33 @@ def verify_typeinfo(
 
 
 def _verify_static_class_methods(
-    stub: nodes.FuncItem, runtime: types.FunctionType
+    stub: nodes.FuncItem, runtime: types.FunctionType, object_path: List[str]
 ) -> Iterator[str]:
-    if isinstance(runtime, classmethod) and not stub.is_class:
+    if runtime.__name__ == "__new__":
+        # Special cased by Python, so never declared as staticmethod
+        return
+    if inspect.isbuiltin(runtime):
+        # The isinstance checks don't work reliably for builtins, e.g. datetime.datetime.now, so do
+        # something a little hacky that seems to work well
+        probably_class_method = isinstance(getattr(runtime, "__self__", None), type)
+        if probably_class_method and not stub.is_class:
+            yield "runtime is a classmethod but stub is not"
+        if not probably_class_method and stub.is_class:
+            yield "stub is a classmethod but runtime is not"
+        return
+
+    # Look the object up statically, to avoid binding by the descriptor protocol
+    static_runtime = importlib.import_module(object_path[0])
+    for entry in object_path[1:]:
+        static_runtime = inspect.getattr_static(static_runtime, entry)
+
+    if isinstance(static_runtime, classmethod) and not stub.is_class:
         yield "runtime is a classmethod but stub is not"
-    if not isinstance(runtime, classmethod) and stub.is_class:
+    if not isinstance(static_runtime, classmethod) and stub.is_class:
         yield "stub is a classmethod but runtime is not"
-    if isinstance(runtime, staticmethod) and not stub.is_static:
+    if isinstance(static_runtime, staticmethod) and not stub.is_static:
         yield "runtime is a staticmethod but stub is not"
-    if not isinstance(runtime, classmethod) and stub.is_static:
+    if not isinstance(static_runtime, staticmethod) and stub.is_static:
         yield "stub is a staticmethod but runtime is not"
 
 
@@ -590,7 +608,7 @@ def verify_funcitem(
         return
 
     for message in _verify_static_class_methods(stub, runtime, object_path):
-        yield Error(object_path, message, stub, runtime)
+        yield Error(object_path, "is inconsistent, " + message, stub, runtime)
 
     try:
         signature = inspect.signature(runtime)

From 0dbd899296ac25ef9eb874b27d4ce5b235212599 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 25 Jan 2020 18:06:49 -0800
Subject: [PATCH 61/74] stubtest: [minor] factor out is_dunder, check suffix

---
 scripts/stubtest.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 9bfee912a628..0eeb10cc81f4 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -274,7 +274,7 @@ def _verify_arg_name(
 ) -> Iterator[str]:
     """Checks whether argument names match."""
     # Ignore exact names for all dunder methods other than __init__
-    if function_name != "__init__" and function_name.startswith("__"):
+    if is_dunder(function_name, exclude_init=True):
         return
 
     def strip_prefix(s: str, prefix: str) -> str:
@@ -439,7 +439,7 @@ def from_overloadedfuncdef(
 
         """
         # For all dunder methods other than __init__, just assume all args are positional-only
-        assume_positional_only = stub.name != "__init__" and stub.name.startswith("__")
+        assume_positional_only = is_dunder(stub.name, exclude_init=True)
 
         all_args: Dict[str, List[Tuple[nodes.Argument, int]]] = {}
         for func in map(_resolve_funcitem_from_decorator, stub.items):
@@ -521,7 +521,7 @@ def _verify_signature(
             runtime_arg.kind == inspect.Parameter.POSITIONAL_ONLY
             and not stub_arg.variable.name.startswith("__")
             and not stub_arg.variable.name.strip("_") == "self"
-            and not function_name.startswith("__")  # noisy for dunder methods
+            and not is_dunder(function_name)  # noisy for dunder methods
         ):
             yield (
                 f'stub argument "{stub_arg.variable.name}" should be positional-only '
@@ -810,6 +810,17 @@ def verify_typealias(
         yield None
 
 
+def is_dunder(name: str, exclude_init: bool = False) -> bool:
+    """Returns whether name is a dunder name.
+
+    :param exclude_init: Whether to return False for __init__
+
+    """
+    if exclude_init and name == "__init__":
+        return False
+    return name.startswith("__") and name.endswith("__")
+
+
 def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool:
     """Checks whether ``left`` is a subtype of ``right``."""
     if (

From 204c1688707b5a2310cc8ade84cbfb5d83ba6a82 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 25 Jan 2020 18:29:02 -0800
Subject: [PATCH 62/74] stubtest: fix proper_plugin, other selfcheck errors

---
 scripts/stubtest.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 0eeb10cc81f4..950af0f19cf7 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -823,6 +823,8 @@ def is_dunder(name: str, exclude_init: bool = False) -> bool:
 
 def is_subtype_helper(left: mypy.types.Type, right: mypy.types.Type) -> bool:
     """Checks whether ``left`` is a subtype of ``right``."""
+    left = mypy.types.get_proper_type(left)
+    right = mypy.types.get_proper_type(right)
     if (
         isinstance(left, mypy.types.LiteralType)
         and isinstance(left.value, int)
@@ -862,7 +864,8 @@ def get_mypy_type_of_runtime_value(runtime: Any) -> Optional[mypy.types.Type]:
     if not isinstance(type_info, nodes.TypeInfo):
         return None
 
-    anytype = lambda: mypy.types.AnyType(mypy.types.TypeOfAny.unannotated)
+    def anytype() -> mypy.types.AnyType:
+        return mypy.types.AnyType(mypy.types.TypeOfAny.unannotated)
 
     if isinstance(runtime, tuple):
         # Special case tuples so we construct a valid mypy.types.TupleType

From 523761cad03a1b86a55dfb2f93a0f3652858f1ff Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 28 Jan 2020 14:23:58 -0800
Subject: [PATCH 63/74] stubtest: find submodules when explicitly testing a
 module

---
 scripts/stubtest.py | 35 +++++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 950af0f19cf7..c9c5a8b3cea6 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -887,24 +887,41 @@ def anytype() -> mypy.types.AnyType:
 _all_stubs: Dict[str, nodes.MypyFile] = {}
 
 
-def build_stubs(modules: List[str], options: Options) -> None:
+def build_stubs(
+    modules: List[str], options: Options, find_submodules: bool = False
+) -> List[str]:
     """Uses mypy to construct stub objects for the given modules.
 
     This sets global state that ``get_stub`` can access.
 
+    Returns all modules we might want to check. If ``find_submodules`` is False, this is equal
+    to ``modules``.
+
+    :param modules: List of modules to build stubs for.
+    :param options: Mypy options for finding and building stubs.
+    :param find_submodules: Whether to attempt to find submodules of the given modules as well.
+
     """
     data_dir = mypy.build.default_data_dir()
     search_path = mypy.modulefinder.compute_search_paths([], options, data_dir)
     find_module_cache = mypy.modulefinder.FindModuleCache(search_path)
 
+    all_modules = []
     sources = []
-    # TODO: restore support for automatically recursing into submodules with find_modules_recursive
     for module in modules:
-        module_path = find_module_cache.find_module(module)
-        if module_path is None:
-            # test_module will yield an error later when it can't find stubs
-            continue
-        sources.append(mypy.modulefinder.BuildSource(module_path, module, None))
+        all_modules.append(module)
+        if not find_submodules:
+            module_path = find_module_cache.find_module(module)
+            if module_path is None:
+                # test_module will yield an error later when it can't find stubs
+                continue
+            sources.append(mypy.modulefinder.BuildSource(module_path, module, None))
+        else:
+            found_sources = find_module_cache.find_modules_recursive(module)
+            sources.extend(found_sources)
+            all_modules.extend(
+                s.module for s in found_sources if s.module not in all_modules
+            )
 
     res = mypy.build.build(sources=sources, options=options)
     if res.errors:
@@ -915,6 +932,8 @@ def build_stubs(modules: List[str], options: Options) -> None:
     global _all_stubs
     _all_stubs = res.files
 
+    return all_modules
+
 
 def get_stub(module: str) -> Optional[nodes.MypyFile]:
     """Returns a stub object for the given module, if we've built one."""
@@ -1021,7 +1040,7 @@ def main() -> int:
     options.incremental = False
     options.custom_typeshed_dir = args.custom_typeshed_dir
 
-    build_stubs(modules, options)
+    modules = build_stubs(modules, options, find_submodules=not args.check_typeshed)
 
     exit_code = 0
     for module in modules:

From 729181a9e450deec1598923ff81765115cd1bbb6 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 28 Jan 2020 14:33:23 -0800
Subject: [PATCH 64/74] stubtest: remove f-strings for py35

---
 scripts/stubtest.py | 85 ++++++++++++++++++++++++++-------------------
 1 file changed, 50 insertions(+), 35 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index c9c5a8b3cea6..1868220c3255 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -97,9 +97,9 @@ def get_description(self, concise: bool = False) -> str:
 
         stub_loc_str = ""
         if stub_line:
-            stub_loc_str += f" at line {stub_line}"
+            stub_loc_str += " at line {}".format(stub_line)
         if stub_file:
-            stub_loc_str += f" in file {stub_file}"
+            stub_loc_str += " in file {}".format(stub_file)
 
         runtime_line = None
         runtime_file = None
@@ -115,14 +115,16 @@ def get_description(self, concise: bool = False) -> str:
 
         runtime_loc_str = ""
         if runtime_line:
-            runtime_loc_str += f" at line {runtime_line}"
+            runtime_loc_str += " at line {}".format(runtime_line)
         if runtime_file:
-            runtime_loc_str += f" in file {runtime_file}"
+            runtime_loc_str += " in file {}".format(runtime_file)
 
         output = [
             _style("error: ", color="red", bold=True),
             _style(self.object_desc, bold=True),
-            f" {self.message}\n",
+            " ",
+            self.message,
+            "\n",
             "Stub:",
             _style(stub_loc_str, dim=True),
             "\n",
@@ -151,7 +153,7 @@ def test_module(module_name: str) -> Iterator[Error]:
     try:
         runtime = importlib.import_module(module_name)
     except Exception as e:
-        yield Error([module_name], f"failed to import: {e}", stub, MISSING)
+        yield Error([module_name], "failed to import: {}".format(e), stub, MISSING)
         return
 
     # collections likes to warn us about the things we're doing
@@ -297,8 +299,9 @@ def names_approx_match(a: str, b: str) -> bool:
     if stub_arg.variable.name == "_self":
         return
     yield (
-        f'stub argument "{stub_arg.variable.name}" differs from '
-        f'runtime argument "{runtime_arg.name}"'
+        'stub argument "{}" differs from runtime argument "{}"'.format(
+            stub_arg.variable.name, runtime_arg.name
+        )
     )
 
 
@@ -309,8 +312,9 @@ def _verify_arg_default_value(
     if runtime_arg.default != inspect.Parameter.empty:
         if stub_arg.kind not in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
             yield (
-                f'runtime argument "{runtime_arg.name}" has a default value '
-                "but stub argument does not"
+                'runtime argument "{}" has a default value but stub argument does not'.format(
+                    runtime_arg.name
+                )
             )
         else:
             runtime_type = get_mypy_type_of_runtime_value(runtime_arg.default)
@@ -327,14 +331,17 @@ def _verify_arg_default_value(
                 and not is_subtype_helper(runtime_type, stub_type)
             ):
                 yield (
-                    f'runtime argument "{runtime_arg.name}" has a default value of type '
-                    f"{runtime_type}, which is incompatible with stub argument type {stub_type}"
+                    'runtime argument "{}" has a default value of type {}, '
+                    "which is incompatible with stub argument type {}".format(
+                        runtime_arg.name, runtime_type, stub_type
+                    )
                 )
     else:
         if stub_arg.kind in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT):
             yield (
-                f'stub argument "{stub_arg.variable.name}" has a default '
-                "value but runtime argument does not"
+                'stub argument "{}" has a default value but runtime argument does not'.format(
+                    stub_arg.variable.name
+                )
             )
 
 
@@ -371,7 +378,7 @@ def get_desc(arg: Any) -> str:
             arg_type = get_type(arg)
             return (
                 get_name(arg)
-                + (f": {arg_type}" if arg_type else "")
+                + (": {}".format(arg_type) if arg_type else "")
                 + (" = ..." if has_default(arg) else "")
             )
 
@@ -448,7 +455,7 @@ def from_overloadedfuncdef(
                 # For positional-only args, we allow overloads to have different names for the same
                 # argument. To accomplish this, we just make up a fake index-based name.
                 name = (
-                    f"__{index}"
+                    "__{}".format(index)
                     if arg.variable.name.startswith("__") or assume_positional_only
                     else arg.variable.name
                 )
@@ -524,25 +531,27 @@ def _verify_signature(
             and not is_dunder(function_name)  # noisy for dunder methods
         ):
             yield (
-                f'stub argument "{stub_arg.variable.name}" should be positional-only '
-                f'(rename with a leading double underscore, i.e. "__{runtime_arg.name}")'
+                'stub argument "{}" should be positional-only '
+                '(rename with a leading double underscore, i.e. "__{}")'.format(
+                    stub_arg.variable.name, runtime_arg.name
+                )
             )
         if (
             runtime_arg.kind != inspect.Parameter.POSITIONAL_ONLY
             and stub_arg.variable.name.startswith("__")
         ):
             yield (
-                f'stub argument "{stub_arg.variable.name}" is positional or keyword '
-                "(remove leading double underscore)"
+                'stub argument "{}" is positional or keyword '
+                "(remove leading double underscore)".format(stub_arg.variable.name)
             )
 
     # Checks involving *args
     if len(stub.pos) == len(runtime.pos):
         if stub.varpos is None and runtime.varpos is not None:
-            yield f'stub does not have *args argument "{runtime.varpos.name}"'
+            yield 'stub does not have *args argument "{}"'.format(runtime.varpos.name)
         if stub.varpos is not None and runtime.varpos is None:
-            yield (
-                f'runtime does not have *args argument "{stub.varpos.variable.name}"'
+            yield 'runtime does not have *args argument "{}"'.format(
+                stub.varpos.variable.name
             )
     elif len(stub.pos) > len(runtime.pos):
         if runtime.varpos is None:
@@ -550,16 +559,20 @@ def _verify_signature(
                 # If the variable is in runtime.kwonly, it's just mislabelled as not a
                 # keyword-only argument; we report the error while checking keyword-only arguments
                 if stub_arg.variable.name not in runtime.kwonly:
-                    yield f'runtime does not have argument "{stub_arg.variable.name}"'
+                    yield 'runtime does not have argument "{}"'.format(
+                        stub_arg.variable.name
+                    )
         # We do not check whether stub takes *args when the runtime does, for cases where the stub
         # just listed out the extra parameters the function takes
     elif len(stub.pos) < len(runtime.pos):
         if stub.varpos is None:
             for runtime_arg in runtime.pos[len(stub.pos) :]:
-                yield f'stub does not have argument "{runtime_arg.name}"'
+                yield 'stub does not have argument "{}"'.format(runtime_arg.name)
         elif runtime.pos is None:
             yield (
-                f'runtime does not have *args argument "{stub.varpos.variable.name}"'
+                'runtime does not have *args argument "{}"'.format(
+                    stub.varpos.variable.name
+                )
             )
 
     # Check keyword-only args
@@ -576,18 +589,20 @@ def _verify_signature(
         # isn't marked as keyword-only
         stub_pos_names = set(stub_arg.variable.name for stub_arg in stub.pos)
         if not set(runtime.kwonly).issubset(set(stub.kwonly) | stub_pos_names):
-            yield f'stub does not have **kwargs argument "{runtime.varkw.name}"'
+            yield 'stub does not have **kwargs argument "{}"'.format(runtime.varkw.name)
     if stub.varkw is not None and runtime.varkw is None:
-        yield f'runtime does not have **kwargs argument "{stub.varkw.variable.name}"'
+        yield 'runtime does not have **kwargs argument "{}"'.format(
+            stub.varkw.variable.name
+        )
     if runtime.varkw is None or not set(runtime.kwonly).issubset(set(stub.kwonly)):
         for arg in sorted(set(stub.kwonly) - set(runtime.kwonly)):
-            yield f'runtime does not have argument "{arg}"'
+            yield 'runtime does not have argument "{}"'.format(arg)
     if stub.varkw is None or not set(stub.kwonly).issubset(set(runtime.kwonly)):
         for arg in sorted(set(runtime.kwonly) - set(stub.kwonly)):
             if arg in set(stub_arg.variable.name for stub_arg in stub.pos):
-                yield f'stub argument "{arg}" is not keyword-only'
+                yield 'stub argument "{}" is not keyword-only'.format(arg)
             else:
-                yield f'stub does not have argument "{arg}"'
+                yield 'stub does not have argument "{}"'.format(arg)
 
 
 @verify.register(nodes.FuncItem)
@@ -663,7 +678,7 @@ def verify_var(
     ):
         yield Error(
             object_path,
-            f"variable differs from runtime type {runtime_type}",
+            "variable differs from runtime type {}".format(runtime_type),
             stub,
             runtime,
         )
@@ -701,7 +716,7 @@ def verify_overloadedfuncdef(
             "is inconsistent, " + message,
             stub,
             runtime,
-            stub_desc=str(stub.type) + f"\nInferred signature: {stub_sig}",
+            stub_desc=str(stub.type) + "\nInferred signature: {}".format(stub_sig),
             runtime_desc="def " + str(signature),
         )
 
@@ -953,7 +968,7 @@ def get_typeshed_stdlib_modules(custom_typeshed_dir: Optional[str]) -> List[str]
 
     versions = ["2and3", "3"]
     for minor in range(sys.version_info.minor + 1):
-        versions.append(f"3.{minor}")
+        versions.append("3.{}".format(minor))
 
     modules = []
     for version in versions:
@@ -1063,7 +1078,7 @@ def main() -> int:
     for w in whitelist:
         if not whitelist[w]:
             exit_code = 1
-            print(f"note: unused whitelist entry {w}")
+            print("note: unused whitelist entry {}".format(w))
 
     # Print the generated whitelist
     if args.generate_whitelist:

From 36163f74c109e973cd159acd477f6cedde239811 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 28 Jan 2020 14:43:17 -0800
Subject: [PATCH 65/74] stubtest: remove variable annotations for py35

---
 scripts/stubtest.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 1868220c3255..d5682049f2b7 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -347,10 +347,10 @@ def _verify_arg_default_value(
 
 class Signature(Generic[T]):
     def __init__(self) -> None:
-        self.pos: List[T] = []
-        self.kwonly: Dict[str, T] = {}
-        self.varpos: Optional[T] = None
-        self.varkw: Optional[T] = None
+        self.pos = []  # type: List[T]
+        self.kwonly = {}  # type: Dict[str, T]
+        self.varpos = None  # type: Optional[T]
+        self.varkw = None  # type: Optional[T]
 
     def __str__(self) -> str:
         def get_name(arg: Any) -> str:
@@ -398,7 +398,7 @@ def get_desc(arg: Any) -> str:
 
     @staticmethod
     def from_funcitem(stub: nodes.FuncItem) -> "Signature[nodes.Argument]":
-        stub_sig: Signature[nodes.Argument] = Signature()
+        stub_sig = Signature()  # type: Signature[nodes.Argument]
         for stub_arg in stub.arguments:
             if stub_arg.kind in (nodes.ARG_POS, nodes.ARG_OPT):
                 stub_sig.pos.append(stub_arg)
@@ -416,7 +416,7 @@ def from_funcitem(stub: nodes.FuncItem) -> "Signature[nodes.Argument]":
     def from_inspect_signature(
         signature: inspect.Signature,
     ) -> "Signature[inspect.Parameter]":
-        runtime_sig: Signature[inspect.Parameter] = Signature()
+        runtime_sig = Signature()  # type: Signature[inspect.Parameter]
         for runtime_arg in signature.parameters.values():
             if runtime_arg.kind in (
                 inspect.Parameter.POSITIONAL_ONLY,
@@ -448,7 +448,7 @@ def from_overloadedfuncdef(
         # For all dunder methods other than __init__, just assume all args are positional-only
         assume_positional_only = is_dunder(stub.name, exclude_init=True)
 
-        all_args: Dict[str, List[Tuple[nodes.Argument, int]]] = {}
+        all_args = {}  # type: Dict[str, List[Tuple[nodes.Argument, int]]]
         for func in map(_resolve_funcitem_from_decorator, stub.items):
             assert func is not None
             for index, arg in enumerate(func.arguments):
@@ -492,7 +492,7 @@ def get_kind(arg_name: str) -> int:
                 return nodes.ARG_OPT if is_pos else nodes.ARG_NAMED_OPT
             return nodes.ARG_POS if is_pos else nodes.ARG_NAMED
 
-        sig: Signature[nodes.Argument] = Signature()
+        sig = Signature()  # type: Signature[nodes.Argument]
         for arg_name in sorted(all_args, key=get_position):
             # example_arg_name gives us a real name (in case we had a fake index-based name)
             example_arg_name = all_args[arg_name][0][0].variable.name
@@ -791,7 +791,7 @@ def apply_decorator_to_funcitem(
         # anything else when running on typeshed's stdlib.
         return None
 
-    func: nodes.FuncItem = dec.func
+    func = dec.func  # type: nodes.FuncItem
     for decorator in dec.original_decorators:
         resulting_func = apply_decorator_to_funcitem(decorator, func)
         if resulting_func is None:
@@ -899,7 +899,7 @@ def anytype() -> mypy.types.AnyType:
     )
 
 
-_all_stubs: Dict[str, nodes.MypyFile] = {}
+_all_stubs = {}  # type: Dict[str, nodes.MypyFile]
 
 
 def build_stubs(

From 63cfe6772f0e575681a49ee17055af6e99b66fad Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 28 Jan 2020 14:51:55 -0800
Subject: [PATCH 66/74] stubtest: remove trailing commas for py35

---
 scripts/stubtest.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index d5682049f2b7..d949c0cf95de 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -56,7 +56,7 @@ def __init__(
         runtime_object: MaybeMissing[Any],
         *,
         stub_desc: Optional[str] = None,
-        runtime_desc: Optional[str] = None,
+        runtime_desc: Optional[str] = None
     ) -> None:
         """Represents an error found by stubtest.
 
@@ -181,7 +181,7 @@ def verify(
 def verify_mypyfile(
     stub: nodes.MypyFile,
     runtime: MaybeMissing[types.ModuleType],
-    object_path: List[str],
+    object_path: List[str]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
         yield Error(object_path, "is not present at runtime", stub, runtime)
@@ -518,7 +518,7 @@ def get_kind(arg_name: str) -> int:
 def _verify_signature(
     stub: Signature[nodes.Argument],
     runtime: Signature[inspect.Parameter],
-    function_name: str,
+    function_name: str
 ) -> Iterator[str]:
     # Check positional arguments match up
     for stub_arg, runtime_arg in zip(stub.pos, runtime.pos):
@@ -609,7 +609,7 @@ def _verify_signature(
 def verify_funcitem(
     stub: nodes.FuncItem,
     runtime: MaybeMissing[types.FunctionType],
-    object_path: List[str],
+    object_path: List[str]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
         yield Error(object_path, "is not present at runtime", stub, runtime)

From c5b99c63207352b5715104b27143c92a5300ac64 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 28 Jan 2020 14:59:51 -0800
Subject: [PATCH 67/74] stubtest: other changes for py35

---
 scripts/stubtest.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index d949c0cf95de..afc5ab70348e 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -655,8 +655,8 @@ def verify_none(
             # weird things going on. Try to see if we can find a runtime object by importing it,
             # otherwise crash.
             runtime = importlib.import_module(".".join(object_path))
-        except ModuleNotFoundError:
-            assert False
+        except ImportError:
+            raise RuntimeError
     yield Error(object_path, "is not present in stub", stub, runtime)
 
 
@@ -974,9 +974,7 @@ def get_typeshed_stdlib_modules(custom_typeshed_dir: Optional[str]) -> List[str]
     for version in versions:
         base = typeshed_dir / "stdlib" / version
         if base.exists():
-            output = subprocess.check_output(
-                ["find", base, "-type", "f"], encoding="utf-8"
-            )
+            output = subprocess.check_output(["find", str(base), "-type", "f"]).decode("utf-8")
             paths = [Path(p) for p in output.splitlines()]
             for path in paths:
                 if path.stem == "__init__":
@@ -1005,7 +1003,7 @@ def strip_comments(s: str) -> str:
 
 
 def main() -> int:
-    assert sys.version_info >= (3, 6), "This script requires at least Python 3.6"
+    assert sys.version_info >= (3, 5), "This script requires at least Python 3.5"
 
     parser = argparse.ArgumentParser()
     parser.add_argument("modules", nargs="*", help="Modules to test")

From b757b22d5cb0d2b7abd6004dfff2a5c1ca74ba77 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 28 Jan 2020 15:08:36 -0800
Subject: [PATCH 68/74] stubtest: [minor] use line length 99 to match project

---
 scripts/stubtest.py | 91 +++++++++++----------------------------------
 1 file changed, 21 insertions(+), 70 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index afc5ab70348e..5ca5bd987a1a 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -179,9 +179,7 @@ def verify(
 
 @verify.register(nodes.MypyFile)
 def verify_mypyfile(
-    stub: nodes.MypyFile,
-    runtime: MaybeMissing[types.ModuleType],
-    object_path: List[str]
+    stub: nodes.MypyFile, runtime: MaybeMissing[types.ModuleType], object_path: List[str]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
         yield Error(object_path, "is not present at runtime", stub, runtime)
@@ -198,9 +196,7 @@ def verify_mypyfile(
     )
     # Check all things declared in module's __all__
     to_check.update(getattr(runtime, "__all__", []))
-    to_check.difference_update(
-        {"__file__", "__doc__", "__name__", "__builtins__", "__package__"}
-    )
+    to_check.difference_update({"__file__", "__doc__", "__name__", "__builtins__", "__package__"})
     # We currently don't check things in the module that aren't in the stub, other than things that
     # are in __all__, to avoid false positives.
 
@@ -217,13 +213,7 @@ def verify_typeinfo(
     stub: nodes.TypeInfo, runtime: MaybeMissing[Type[Any]], object_path: List[str]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
-        yield Error(
-            object_path,
-            "is not present at runtime",
-            stub,
-            runtime,
-            stub_desc=repr(stub),
-        )
+        yield Error(object_path, "is not present at runtime", stub, runtime, stub_desc=repr(stub))
         return
     if not isinstance(runtime, type):
         yield Error(object_path, "is not a type", stub, runtime, stub_desc=repr(stub))
@@ -385,11 +375,7 @@ def get_desc(arg: Any) -> str:
         ret = "def ("
         ret += ", ".join(
             [get_desc(arg) for arg in self.pos]
-            + (
-                ["*" + get_name(self.varpos)]
-                if self.varpos
-                else (["*"] if self.kwonly else [])
-            )
+            + (["*" + get_name(self.varpos)] if self.varpos else (["*"] if self.kwonly else []))
             + [get_desc(arg) for arg in self.kwonly.values()]
             + (["**" + get_name(self.varkw)] if self.varkw else [])
         )
@@ -413,9 +399,7 @@ def from_funcitem(stub: nodes.FuncItem) -> "Signature[nodes.Argument]":
         return stub_sig
 
     @staticmethod
-    def from_inspect_signature(
-        signature: inspect.Signature,
-    ) -> "Signature[inspect.Parameter]":
+    def from_inspect_signature(signature: inspect.Signature,) -> "Signature[inspect.Parameter]":
         runtime_sig = Signature()  # type: Signature[inspect.Parameter]
         for runtime_arg in signature.parameters.values():
             if runtime_arg.kind in (
@@ -434,9 +418,7 @@ def from_inspect_signature(
         return runtime_sig
 
     @staticmethod
-    def from_overloadedfuncdef(
-        stub: nodes.OverloadedFuncDef,
-    ) -> "Signature[nodes.Argument]":
+    def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef,) -> "Signature[nodes.Argument]":
         """Returns a Signature from an OverloadedFuncDef.
 
         If life were simple, to verify_overloadedfuncdef, we'd just verify_funcitem for each of its
@@ -468,8 +450,7 @@ def get_position(arg_name: str) -> int:
         def get_type(arg_name: str) -> mypy.types.ProperType:
             with mypy.state.strict_optional_set(True):
                 all_types = [
-                    arg.variable.type or arg.type_annotation
-                    for arg, _ in all_args[arg_name]
+                    arg.variable.type or arg.type_annotation for arg, _ in all_args[arg_name]
                 ]
                 return mypy.typeops.make_simplified_union([t for t in all_types if t])
 
@@ -516,9 +497,7 @@ def get_kind(arg_name: str) -> int:
 
 
 def _verify_signature(
-    stub: Signature[nodes.Argument],
-    runtime: Signature[inspect.Parameter],
-    function_name: str
+    stub: Signature[nodes.Argument], runtime: Signature[inspect.Parameter], function_name: str
 ) -> Iterator[str]:
     # Check positional arguments match up
     for stub_arg, runtime_arg in zip(stub.pos, runtime.pos):
@@ -550,18 +529,14 @@ def _verify_signature(
         if stub.varpos is None and runtime.varpos is not None:
             yield 'stub does not have *args argument "{}"'.format(runtime.varpos.name)
         if stub.varpos is not None and runtime.varpos is None:
-            yield 'runtime does not have *args argument "{}"'.format(
-                stub.varpos.variable.name
-            )
+            yield 'runtime does not have *args argument "{}"'.format(stub.varpos.variable.name)
     elif len(stub.pos) > len(runtime.pos):
         if runtime.varpos is None:
             for stub_arg in stub.pos[len(runtime.pos) :]:
                 # If the variable is in runtime.kwonly, it's just mislabelled as not a
                 # keyword-only argument; we report the error while checking keyword-only arguments
                 if stub_arg.variable.name not in runtime.kwonly:
-                    yield 'runtime does not have argument "{}"'.format(
-                        stub_arg.variable.name
-                    )
+                    yield 'runtime does not have argument "{}"'.format(stub_arg.variable.name)
         # We do not check whether stub takes *args when the runtime does, for cases where the stub
         # just listed out the extra parameters the function takes
     elif len(stub.pos) < len(runtime.pos):
@@ -569,11 +544,7 @@ def _verify_signature(
             for runtime_arg in runtime.pos[len(stub.pos) :]:
                 yield 'stub does not have argument "{}"'.format(runtime_arg.name)
         elif runtime.pos is None:
-            yield (
-                'runtime does not have *args argument "{}"'.format(
-                    stub.varpos.variable.name
-                )
-            )
+            yield 'runtime does not have *args argument "{}"'.format(stub.varpos.variable.name)
 
     # Check keyword-only args
     for arg in sorted(set(stub.kwonly) & set(runtime.kwonly)):
@@ -591,9 +562,7 @@ def _verify_signature(
         if not set(runtime.kwonly).issubset(set(stub.kwonly) | stub_pos_names):
             yield 'stub does not have **kwargs argument "{}"'.format(runtime.varkw.name)
     if stub.varkw is not None and runtime.varkw is None:
-        yield 'runtime does not have **kwargs argument "{}"'.format(
-            stub.varkw.variable.name
-        )
+        yield 'runtime does not have **kwargs argument "{}"'.format(stub.varkw.variable.name)
     if runtime.varkw is None or not set(runtime.kwonly).issubset(set(stub.kwonly)):
         for arg in sorted(set(stub.kwonly) - set(runtime.kwonly)):
             yield 'runtime does not have argument "{}"'.format(arg)
@@ -607,9 +576,7 @@ def _verify_signature(
 
 @verify.register(nodes.FuncItem)
 def verify_funcitem(
-    stub: nodes.FuncItem,
-    runtime: MaybeMissing[types.FunctionType],
-    object_path: List[str]
+    stub: nodes.FuncItem, runtime: MaybeMissing[types.FunctionType], object_path: List[str]
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
         yield Error(object_path, "is not present at runtime", stub, runtime)
@@ -740,9 +707,7 @@ def _verify_property(stub: nodes.Decorator, runtime: Any) -> Iterator[str]:
     # are read only. So whitelist if runtime_type matches the return type of stub.
     runtime_type = get_mypy_type_of_runtime_value(runtime)
     func_type = (
-        stub.func.type.ret_type
-        if isinstance(stub.func.type, mypy.types.CallableType)
-        else None
+        stub.func.type.ret_type if isinstance(stub.func.type, mypy.types.CallableType) else None
     )
     if (
         runtime_type is not None
@@ -753,9 +718,7 @@ def _verify_property(stub: nodes.Decorator, runtime: Any) -> Iterator[str]:
     yield "is inconsistent, cannot reconcile @property on stub with runtime object"
 
 
-def _resolve_funcitem_from_decorator(
-    dec: nodes.OverloadPart
-) -> Optional[nodes.FuncItem]:
+def _resolve_funcitem_from_decorator(dec: nodes.OverloadPart) -> Optional[nodes.FuncItem]:
     """Returns a FuncItem that corresponds to the output of the decorator.
 
     Returns None if we can't figure out what that would be. For convenience, this function also
@@ -893,18 +856,14 @@ def anytype() -> mypy.types.AnyType:
     # seems to work fine
     return mypy.types.LiteralType(
         value=runtime,
-        fallback=mypy.types.Instance(
-            type_info, [anytype() for _ in type_info.type_vars]
-        ),
+        fallback=mypy.types.Instance(type_info, [anytype() for _ in type_info.type_vars]),
     )
 
 
 _all_stubs = {}  # type: Dict[str, nodes.MypyFile]
 
 
-def build_stubs(
-    modules: List[str], options: Options, find_submodules: bool = False
-) -> List[str]:
+def build_stubs(modules: List[str], options: Options, find_submodules: bool = False) -> List[str]:
     """Uses mypy to construct stub objects for the given modules.
 
     This sets global state that ``get_stub`` can access.
@@ -934,9 +893,7 @@ def build_stubs(
         else:
             found_sources = find_module_cache.find_modules_recursive(module)
             sources.extend(found_sources)
-            all_modules.extend(
-                s.module for s in found_sources if s.module not in all_modules
-            )
+            all_modules.extend(s.module for s in found_sources if s.module not in all_modules)
 
     res = mypy.build.build(sources=sources, options=options)
     if res.errors:
@@ -979,9 +936,7 @@ def get_typeshed_stdlib_modules(custom_typeshed_dir: Optional[str]) -> List[str]
             for path in paths:
                 if path.stem == "__init__":
                     path = path.parent
-                modules.append(
-                    ".".join(path.relative_to(base).parts[:-1] + (path.stem,))
-                )
+                modules.append(".".join(path.relative_to(base).parts[:-1] + (path.stem,)))
     return sorted(modules)
 
 
@@ -1008,9 +963,7 @@ def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("modules", nargs="*", help="Modules to test")
     parser.add_argument(
-        "--check-typeshed",
-        action="store_true",
-        help="Check all stdlib modules in typeshed",
+        "--check-typeshed", action="store_true", help="Check all stdlib modules in typeshed"
     )
     parser.add_argument(
         "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
@@ -1041,9 +994,7 @@ def main() -> int:
 
     modules = args.modules
     if args.check_typeshed:
-        assert (
-            not args.modules
-        ), "Cannot pass both --check-typeshed and a list of modules"
+        assert not args.modules, "Cannot pass both --check-typeshed and a list of modules"
         modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir)
         modules.remove("antigravity")  # it's super annoying
 

From e3f4cd3a572519997733f0b3b05518afcdeca020 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Tue, 28 Jan 2020 16:06:09 -0800
Subject: [PATCH 69/74] stubtest: add a flag to ignore positional-only errors

---
 scripts/stubtest.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 5ca5bd987a1a..b75d26a42357 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -80,6 +80,11 @@ def is_missing_stub(self) -> bool:
         """Whether or not the error is for something missing from the stub."""
         return isinstance(self.stub_object, Missing)
 
+    def is_positional_only_related(self) -> bool:
+        """Whether or not the error is for something being (or not being) positional-only."""
+        # TODO: This is hacky, use error codes or something more resilient
+        return "leading double underscore" in self.message
+
     def get_description(self, concise: bool = False) -> str:
         """Returns a description of the error.
 
@@ -520,7 +525,7 @@ def _verify_signature(
             and stub_arg.variable.name.startswith("__")
         ):
             yield (
-                'stub argument "{}" is positional or keyword '
+                'stub argument "{}" should be positional or keyword '
                 "(remove leading double underscore)".format(stub_arg.variable.name)
             )
 
@@ -973,6 +978,11 @@ def main() -> int:
         action="store_true",
         help="Ignore errors for stub missing things that are present at runtime",
     )
+    parser.add_argument(
+        "--ignore-positional-only",
+        action="store_true",
+        help="Ignore errors for whether an argument should or shouldn't be positional-only",
+    )
     parser.add_argument(
         "--whitelist",
         help="Use file as a whitelist. Whitelists can be created with --generate-whitelist",
@@ -1012,6 +1022,8 @@ def main() -> int:
             # Filter errors
             if args.ignore_missing_stub and error.is_missing_stub():
                 continue
+            if args.ignore_positional_only and error.is_positional_only_related():
+                continue
             if error.object_desc in whitelist:
                 whitelist[error.object_desc] = True
                 continue

From 6efc27f07ab65979e3204547d29728b857ccaef6 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 29 Jan 2020 23:59:15 -0800
Subject: [PATCH 70/74] stubtest: check typevar upper bounds for default values

is_subtype would always return False, leading to false positives most
times TypeVars were used for parameters with default values. We still
have some false positives from Unions of TypeVars, but that's less bad,
and could almost all be fixed by adjusting the overload of Mapping.get
---
 scripts/stubtest.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index b75d26a42357..bf6647e5d8f6 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -318,6 +318,8 @@ def _verify_arg_default_value(
             # UnboundTypes have ugly question marks following them, so default to var type.
             # Note we do this same fallback when constructing signatures in from_overloadedfuncdef
             stub_type = stub_arg.variable.type or stub_arg.type_annotation
+            if isinstance(stub_type, mypy.types.TypeVarType):
+                stub_type = stub_type.upper_bound
             if (
                 runtime_type is not None
                 and stub_type is not None

From dd07f7880d611c3d6ca6184ec8e494a84fc78f48 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 30 Jan 2020 18:14:38 -0800
Subject: [PATCH 71/74] stubtest: don't crash because of bpo-39504

---
 scripts/stubtest.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index bf6647e5d8f6..fffd8d3966a1 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -601,8 +601,9 @@ def verify_funcitem(
 
     try:
         signature = inspect.signature(runtime)
-    except ValueError:
+    except (ValueError, RuntimeError):
         # inspect.signature throws sometimes
+        # catch RuntimeError because of https://bugs.python.org/issue39504
         return
 
     stub_sig = Signature.from_funcitem(stub)

From 13627d96e15e79860ce69d848d97cee86b8861f5 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 31 Jan 2020 17:34:27 -0800
Subject: [PATCH 72/74] stubtest: avoid false positive when defining enums

---
 scripts/stubtest.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index fffd8d3966a1..c03d78d71857 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -6,6 +6,7 @@
 
 import argparse
 import copy
+import enum
 import importlib
 import inspect
 import subprocess
@@ -651,12 +652,21 @@ def verify_var(
         and stub.type is not None
         and not is_subtype_helper(runtime_type, stub.type)
     ):
-        yield Error(
-            object_path,
-            "variable differs from runtime type {}".format(runtime_type),
-            stub,
-            runtime,
-        )
+        should_error = True
+        # Avoid errors when defining enums, since runtime_type is the enum itself, but we'd
+        # annotate it with the type of runtime.value
+        if isinstance(runtime, enum.Enum):
+            runtime_type = get_mypy_type_of_runtime_value(runtime.value)
+            if runtime_type is not None and is_subtype_helper(runtime_type, stub.type):
+                should_error = False
+
+        if should_error:
+            yield Error(
+                object_path,
+                "variable differs from runtime type {}".format(runtime_type),
+                stub,
+                runtime,
+            )
 
 
 @verify.register(nodes.OverloadedFuncDef)

From 6f2cb39115b0fd1d099cd7148c9aff9ca718c885 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 31 Jan 2020 19:21:38 -0800
Subject: [PATCH 73/74] stubtest: allow multiple whitelists

---
 scripts/stubtest.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index c03d78d71857..8bc500735afb 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -998,7 +998,13 @@ def main() -> int:
     )
     parser.add_argument(
         "--whitelist",
-        help="Use file as a whitelist. Whitelists can be created with --generate-whitelist",
+        action="append",
+        metavar="FILE",
+        default=[],
+        help=(
+            "Use file as a whitelist. Can be passed multiple times to combine multiple "
+            "whitelists. Whitelist can be created with --generate-whitelist"
+        ),
     )
     parser.add_argument("--concise", action="store_true", help="Make output concise")
     parser.add_argument(
@@ -1010,7 +1016,11 @@ def main() -> int:
 
     # Load the whitelist. This is a series of strings corresponding to Error.object_desc
     # Values in the dict will store whether we used the whitelist entry or not.
-    whitelist = {entry: False for entry in get_whitelist_entries(args.whitelist)}
+    whitelist = {
+        entry: False
+        for whitelist_file in args.whitelist
+        for entry in get_whitelist_entries(whitelist_file)
+    }
 
     # If we need to generate a whitelist, we store Error.object_desc for each error here.
     generated_whitelist = set()

From 0b89ff9e12e7b12bc06ff5ee8a124d8fb6348317 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 31 Jan 2020 19:29:19 -0800
Subject: [PATCH 74/74] stubtest: [minor] improve help message

---
 scripts/stubtest.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/scripts/stubtest.py b/scripts/stubtest.py
index 8bc500735afb..00475b78168d 100644
--- a/scripts/stubtest.py
+++ b/scripts/stubtest.py
@@ -978,14 +978,11 @@ def strip_comments(s: str) -> str:
 def main() -> int:
     assert sys.version_info >= (3, 5), "This script requires at least Python 3.5"
 
-    parser = argparse.ArgumentParser()
-    parser.add_argument("modules", nargs="*", help="Modules to test")
-    parser.add_argument(
-        "--check-typeshed", action="store_true", help="Check all stdlib modules in typeshed"
-    )
-    parser.add_argument(
-        "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
+    parser = argparse.ArgumentParser(
+        description="Compares stubs to objects introspected from the runtime."
     )
+    parser.add_argument("modules", nargs="*", help="Modules to test")
+    parser.add_argument("--concise", action="store_true", help="Make output concise")
     parser.add_argument(
         "--ignore-missing-stub",
         action="store_true",
@@ -996,6 +993,12 @@ def main() -> int:
         action="store_true",
         help="Ignore errors for whether an argument should or shouldn't be positional-only",
     )
+    parser.add_argument(
+        "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
+    )
+    parser.add_argument(
+        "--check-typeshed", action="store_true", help="Check all stdlib modules in typeshed"
+    )
     parser.add_argument(
         "--whitelist",
         action="append",
@@ -1006,7 +1009,6 @@ def main() -> int:
             "whitelists. Whitelist can be created with --generate-whitelist"
         ),
     )
-    parser.add_argument("--concise", action="store_true", help="Make output concise")
     parser.add_argument(
         "--generate-whitelist",
         action="store_true",