regr_test.py: Allow non-types dependencies

Author: AlexWaygood

tests/mypy_test.py

@@ -274,7 +274,7 @@ def add_third_party_files(
     seen_dists.add(distribution)
 
     stubs_dir = Path("stubs")
-    dependencies = get_recursive_requirements(distribution)
+    dependencies = get_recursive_requirements(distribution).typeshed_pkgs
 
     for dependency in dependencies:
         if dependency in seen_dists:

tests/regr_test.py

@@ -18,7 +18,9 @@
     PackageInfo,
     colored,
     get_all_testcase_directories,
+    get_mypy_req,
     get_recursive_requirements,
+    make_venv,
     print_error,
     print_success_msg,
     testcase_dir_from_package_name,
@@ -85,14 +87,66 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
 )
 
 
-def test_testcase_directory(package: PackageInfo, version: str, platform: str, quiet: bool) -> ReturnCode:
-    package_name, test_case_directory = package
-    is_stdlib = package_name == "stdlib"
+def run_testcases(
+    package: PackageInfo, flags: list[str], tmpdir_path: Path, python_minor_version: int
+) -> tuple[Path, subprocess.CompletedProcess[str]]:
+    python_exe = sys.executable
+    new_test_case_dir = tmpdir_path / "test_cases"
+    shutil.copytree(package.test_case_directory, new_test_case_dir)
+    env_vars = dict(os.environ)
+    if package.is_stdlib:
+        flags.extend(["--no-site-packages", "--custom-typeshed-dir", str(Path(__file__).parent.parent)])
+    else:
+        # HACK: we want to run these test cases in an isolated environment --
+        # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml
+        # (and all stub packages required by those stub packages, etc. etc.),
+        # but none of the other stubs in typeshed.
+        #
+        # The best way of doing that without stopping --warn-unused-ignore from working
+        # seems to be to create a "new typeshed" directory in a tempdir
+        # that has only the required stubs copied over.
+        new_typeshed = tmpdir_path / "typeshed"
+        new_typeshed.mkdir()
+        shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
+        requirements = get_recursive_requirements(package.name)
+        # mypy refuses to consider a directory a "valid typeshed directory"
+        # unless there's a stubs/mypy-extensions path inside it,
+        # so add that to the list of stubs to copy over to the new directory
+        for requirement in set(requirements.typeshed_pkgs) | {package.name, "mypy-extensions"}:
+            shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
+
+        if requirements.external_pkgs:
+            pip_exe, python_exe = make_venv(tmpdir_path / ".venv")
+            try:
+                subprocess.run([pip_exe, "install", get_mypy_req(), *requirements.external_pkgs], check=True, capture_output=True)
+            except subprocess.CalledProcessError as e:
+                print(e.stderr)
+                raise
+        else:
+            flags.append("--no-site-packages")
+
+        env_vars["MYPYPATH"] = os.pathsep.join(map(str, new_typeshed.glob("stubs/*")))
+        flags.extend(["--custom-typeshed-dir", str(new_typeshed)])
+
+    # If the test-case filename ends with -py39,
+    # only run the test if --python-version was set to 3.9 or higher (for example)
+    for path in new_test_case_dir.rglob("*.py"):
+        if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem):
+            minor_version_required = int(match[1])
+            assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS
+            if minor_version_required <= python_minor_version:
+                flags.append(str(path))
+        else:
+            flags.append(str(path))
+
+    return new_test_case_dir, subprocess.run([python_exe, "-m", "mypy", *flags], capture_output=True, text=True, env=env_vars)
 
+
+def test_testcase_directory(package: PackageInfo, version: str, platform: str, quiet: bool) -> ReturnCode:
     msg = f"Running mypy --platform {platform} --python-version {version} on the "
-    msg += "standard library test cases..." if is_stdlib else f"test cases for {package_name!r}..."
+    msg += "standard library test cases..." if package.is_stdlib else f"test cases for {package.name!r}..."
     if not quiet:
-        print(msg, end=" ")
+        print(msg, end=" ", flush=True)
 
     # "--enable-error-code ignore-without-code" is purposefully ommited. See https://github.com/python/typeshed/pull/8083
     flags = [
@@ -103,53 +157,17 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, q
         "--no-error-summary",
         "--platform",
         platform,
-        "--no-site-packages",
         "--strict",
         "--pretty",
     ]
 
     # --warn-unused-ignores doesn't work for files inside typeshed.
-    # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory.
+    # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory,
+    # and run the test cases inside of that.
     with tempfile.TemporaryDirectory() as td:
-        td_path = Path(td)
-        new_test_case_dir = td_path / "test_cases"
-        shutil.copytree(test_case_directory, new_test_case_dir)
-        env_vars = dict(os.environ)
-        if is_stdlib:
-            flags.extend(["--custom-typeshed-dir", str(Path(__file__).parent.parent)])
-        else:
-            # HACK: we want to run these test cases in an isolated environment --
-            # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml
-            # (and all stub packages required by those stub packages, etc. etc.),
-            # but none of the other stubs in typeshed.
-            #
-            # The best way of doing that without stopping --warn-unused-ignore from working
-            # seems to be to create a "new typeshed" directory in a tempdir
-            # that has only the required stubs copied over.
-            new_typeshed = td_path / "typeshed"
-            os.mkdir(new_typeshed)
-            shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
-            requirements = get_recursive_requirements(package_name)
-            # mypy refuses to consider a directory a "valid typeshed directory"
-            # unless there's a stubs/mypy-extensions path inside it,
-            # so add that to the list of stubs to copy over to the new directory
-            for requirement in requirements + ["mypy-extensions"]:
-                shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
-            env_vars["MYPYPATH"] = os.pathsep.join(map(str, new_typeshed.glob("stubs/*")))
-            flags.extend(["--custom-typeshed-dir", str(td_path / "typeshed")])
-
-        # If the test-case filename ends with -py39,
-        # only run the test if --python-version was set to 3.9 or higher (for example)
-        for path in new_test_case_dir.rglob("*.py"):
-            if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem):
-                minor_version_required = int(match[1])
-                assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS
-                if minor_version_required <= int(version.split(".")[1]):
-                    flags.append(str(path))
-            else:
-                flags.append(str(path))
-
-        result = subprocess.run([sys.executable, "-m", "mypy", *flags], capture_output=True, env=env_vars)
+        new_test_case_dir, result = run_testcases(
+            package=package, flags=flags, tmpdir_path=Path(td), python_minor_version=int(version.split(".")[1])
+        )
 
     if result.returncode:
         if quiet:
@@ -158,11 +176,11 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, q
             # If there are errors, the output is inscrutable if this isn't printed.
             print(msg, end=" ")
         print_error("failure\n")
-        replacements = (str(new_test_case_dir), str(test_case_directory))
+        replacements = (str(new_test_case_dir), str(package.test_case_directory))
         if result.stderr:
-            print_error(result.stderr.decode(), fix_path=replacements)
+            print_error(result.stderr, fix_path=replacements)
         if result.stdout:
-            print_error(result.stdout.decode(), fix_path=replacements)
+            print_error(result.stdout, fix_path=replacements)
     elif not quiet:
         print_success_msg()
     return result.returncode

tests/stubtest_third_party.py

@@ -4,23 +4,15 @@
 from __future__ import annotations
 
 import argparse
-import functools
 import os
 import subprocess
 import sys
 import tempfile
-import venv
 from pathlib import Path
 from typing import NoReturn
 
 import tomli
-from utils import colored, print_error, print_success_msg
-
-
-@functools.lru_cache()
-def get_mypy_req() -> str:
-    with open("requirements-tests.txt", encoding="UTF-8") as f:
-        return next(line.strip() for line in f if "mypy" in line)
+from utils import colored, get_mypy_req, make_venv, print_error, print_success_msg
 
 
 def run_stubtest(dist: Path, *, verbose: bool = False, specified_stubs_only: bool = False) -> bool:
@@ -44,25 +36,10 @@ def run_stubtest(dist: Path, *, verbose: bool = False, specified_stubs_only: boo
     with tempfile.TemporaryDirectory() as tmp:
         venv_dir = Path(tmp)
         try:
-            venv.create(venv_dir, with_pip=True, clear=True)
-        except subprocess.CalledProcessError as e:
-            if "ensurepip" in e.cmd:
-                print_error("fail")
-                print_error(
-                    "stubtest requires a Python installation with ensurepip. "
-                    "If on Linux, you may need to install the python3-venv package."
-                )
+            pip_exe, python_exe = make_venv(venv_dir)
+        except Exception:
+            print_error("fail")
             raise
-
-        if sys.platform == "win32":
-            pip = venv_dir / "Scripts" / "pip.exe"
-            python = venv_dir / "Scripts" / "python.exe"
-        else:
-            pip = venv_dir / "bin" / "pip"
-            python = venv_dir / "bin" / "python"
-
-        pip_exe, python_exe = str(pip), str(python)
-
         dist_version = metadata["version"]
         extras = stubtest_meta.get("extras", [])
         assert isinstance(dist_version, str)

tests/utils.py

@@ -4,13 +4,17 @@
 
 import os
 import re
+import subprocess
+import sys
+import venv
 from functools import cache
-from itertools import filterfalse
 from pathlib import Path
 from typing import NamedTuple
+from typing_extensions import Annotated
 
 import pathspec  # type: ignore[import]
 import tomli
+from packaging.requirements import Requirement
 
 # Used to install system-wide packages for different OS types:
 METADATA_MAPPING = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"}
@@ -45,25 +49,82 @@ def print_success_msg() -> None:
 # ====================================================================
 
 
+class PackageDependencies(NamedTuple):
+    typeshed_pkgs: tuple[str, ...]
+    external_pkgs: tuple[str, ...]
+
+
 @cache
-def read_dependencies(distribution: str) -> tuple[str, ...]:
+def read_dependencies(distribution: str) -> PackageDependencies:
     with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
         data = tomli.load(f)
-    requires = data.get("requires", [])
-    assert isinstance(requires, list)
-    dependencies = []
-    for dependency in requires:
-        assert isinstance(dependency, str)
-        assert dependency.startswith("types-"), f"unrecognized dependency {dependency!r}"
-        dependencies.append(dependency[6:].split("<")[0])
-    return tuple(dependencies)
+    dependencies = data.get("requires", [])
+    assert isinstance(dependencies, list)
+    typeshed, external = [], []
+    for dependency in dependencies:
+        if dependency.startswith("types-"):
+            maybe_typeshed_dependency = Requirement(dependency).name.removeprefix("types-")
+            if maybe_typeshed_dependency in os.listdir("stubs"):
+                typeshed.append(maybe_typeshed_dependency)
+            else:
+                external.append(dependency)
+        else:
+            external.append(dependency)
+    return PackageDependencies(tuple(typeshed), tuple(external))
+
+
+def get_recursive_requirements(package_name: str, seen: set[str] | None = None) -> PackageDependencies:
+    typeshed: set[str] = set()
+    external: set[str] = set()
+    seen = seen if seen is not None else {package_name}
+    non_recursive_requirements = read_dependencies(package_name)
+    typeshed.update(non_recursive_requirements.typeshed_pkgs)
+    external.update(non_recursive_requirements.external_pkgs)
+    for pkg in non_recursive_requirements.typeshed_pkgs:
+        if pkg in seen:
+            continue
+        reqs = get_recursive_requirements(pkg)
+        typeshed.update(reqs.typeshed_pkgs)
+        external.update(reqs.external_pkgs)
+        seen.add(pkg)
+    return PackageDependencies(tuple(sorted(typeshed)), tuple(sorted(external)))
 
 
-def get_recursive_requirements(package_name: str, seen: set[str] | None = None) -> list[str]:
-    seen = seen if seen is not None else {package_name}
-    for dependency in filterfalse(seen.__contains__, read_dependencies(package_name)):
-        seen.update(get_recursive_requirements(dependency, seen))
-    return sorted(seen | {package_name})
+# ====================================================================
+# Dynamic venv creation
+# ====================================================================
+
+
+class VenvInfo(NamedTuple):
+    pip_exe: Annotated[str, "A path to the venv's pip executable"]
+    python_exe: Annotated[str, "A path to the venv's python executable"]
+
+
+def make_venv(venv_dir: Path) -> VenvInfo:
+    try:
+        venv.create(venv_dir, with_pip=True, clear=True)
+    except subprocess.CalledProcessError as e:
+        if "ensurepip" in e.cmd:
+            print_error(
+                "stubtest requires a Python installation with ensurepip. "
+                "If on Linux, you may need to install the python3-venv package."
+            )
+        raise
+
+    if sys.platform == "win32":
+        pip = venv_dir / "Scripts" / "pip.exe"
+        python = venv_dir / "Scripts" / "python.exe"
+    else:
+        pip = venv_dir / "bin" / "pip"
+        python = venv_dir / "bin" / "python"
+
+    return VenvInfo(str(pip), str(python))
+
+
+@cache
+def get_mypy_req() -> str:
+    with open("requirements-tests.txt", encoding="UTF-8") as f:
+        return next(line.strip() for line in f if "mypy" in line)
 
 
 # ====================================================================
@@ -83,6 +144,10 @@ class PackageInfo(NamedTuple):
     name: str
     test_case_directory: Path
 
+    @property
+    def is_stdlib(self) -> bool:
+        return self.name == "stdlib"
+
 
 def testcase_dir_from_package_name(package_name: str) -> Path:
     return Path("stubs", package_name, "@tests/test_cases")