stubtest: fallback to dir if runtime doesn't have __all__ (#9523)

Co-authored-by: hauntsaninja <>

Author: hauntsaninja

mypy/stubtest.py

@@ -212,11 +212,18 @@ def verify_mypyfile(
         for m, o in stub.names.items()
         if o.module_public and (not m.startswith("_") or hasattr(runtime, m))
     )
-    # Check all things declared in module's __all__
-    to_check.update(getattr(runtime, "__all__", []))
+    runtime_public_contents = [
+        m
+        for m in dir(runtime)
+        if not m.startswith("_")
+        # Ensure that the object's module is `runtime`, e.g. so that we don't pick up reexported
+        # modules and infinitely recurse. Unfortunately, there's no way to detect an explicit
+        # reexport missing from the stubs (that isn't specified in __all__)
+        and getattr(getattr(runtime, m), "__module__", None) == runtime.__name__
+    ]
+    # Check all things declared in module's __all__, falling back to runtime_public_contents
+    to_check.update(getattr(runtime, "__all__", runtime_public_contents))
     to_check.difference_update({"__file__", "__doc__", "__name__", "__builtins__", "__package__"})
-    # We currently don't check things in the module that aren't in the stub, other than things that
-    # are in __all__, to avoid false positives.
 
     for entry in sorted(to_check):
         yield from verify(

mypy/test/teststubtest.py

@@ -69,9 +69,16 @@ def collect_cases(fn: Callable[..., Iterator[Case]]) -> Callable[..., None]:
 
     def test(*args: Any, **kwargs: Any) -> None:
         cases = list(fn(*args, **kwargs))
-        expected_errors = set(
-            "{}.{}".format(TEST_MODULE_NAME, c.error) for c in cases if c.error is not None
-        )
+        expected_errors = set()
+        for c in cases:
+            if c.error is None:
+                continue
+            expected_error = "{}.{}".format(TEST_MODULE_NAME, c.error)
+            assert expected_error not in expected_errors, (
+                "collect_cases merges cases into a single stubtest invocation; we already "
+                "expect an error for {}".format(expected_error)
+            )
+            expected_errors.add(expected_error)
         output = run_stubtest(
             stub="\n\n".join(textwrap.dedent(c.stub.lstrip("\n")) for c in cases),
             runtime="\n\n".join(textwrap.dedent(c.runtime.lstrip("\n")) for c in cases),
@@ -582,6 +589,11 @@ def h(x: str): ...
             stub="from mystery import A, B as B, C as D  # type: ignore", runtime="", error="B"
         )
 
+    @collect_cases
+    def test_missing_no_runtime_all(self) -> Iterator[Case]:
+        yield Case(stub="", runtime="import sys", error=None)
+        yield Case(stub="", runtime="def g(): ...", error="g")
+
     @collect_cases
     def test_name_mangling(self) -> Iterator[Case]:
         yield Case(
@@ -666,6 +678,11 @@ def test_ignore_flags(self) -> None:
         )
         assert not output
 
+        output = run_stubtest(
+            stub="", runtime="def f(): pass", options=["--ignore-missing-stub"]
+        )
+        assert not output
+
         output = run_stubtest(
             stub="def f(__a): ...", runtime="def f(a): pass", options=["--ignore-positional-only"]
         )