From 770e39fff735785a22edc5dc8647ef4d3e2babf2 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Fri, 15 Sep 2023 19:57:29 +0100 Subject: [PATCH 1/2] Add a custom script for running mypy on libregrtest --- Lib/test/libregrtest/mypy.ini | 1 + Lib/test/support/__init__.py | 27 ++++++++---- Tools/scripts/run_mypy_on_regrtest.py | 59 +++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 Tools/scripts/run_mypy_on_regrtest.py diff --git a/Lib/test/libregrtest/mypy.ini b/Lib/test/libregrtest/mypy.ini index ac2f70c2c1f3ab..21fdca6df060a4 100644 --- a/Lib/test/libregrtest/mypy.ini +++ b/Lib/test/libregrtest/mypy.ini @@ -45,3 +45,4 @@ ignore_missing_imports = True [mypy-test.*] ignore_missing_imports = True +disable_error_code = attr-defined,var-annotated diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 0d9efe10c6504c..6b4102b6d6d053 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -18,6 +18,7 @@ import types import unittest import warnings +from collections.abc import Callable from .testresult import get_test_runner @@ -183,12 +184,12 @@ def get_attribute(obj, name): else: return attribute -verbose = 1 # Flag set to 0 by regrtest.py -use_resources = None # Flag set to [] by regrtest.py -max_memuse = 0 # Disable bigmem tests (they will still be run with - # small sizes, to make sure they work.) +verbose = 1 # Flag set to 0 by regrtest.py +use_resources: tuple[str, ...] | None = None # Flag set to a tuple by regrtest.py +max_memuse = 0 # Disable bigmem tests (they will still be run with + # small sizes, to make sure they work.) real_max_memuse = 0 -junit_xml_list = None # list of testsuite XML elements +junit_xml_list: list | None = None # list of testsuite XML elements failfast = False # _original_stdout is meant to hold stdout at the time regrtest began. @@ -1325,6 +1326,18 @@ def flush_std_streams(): sys.stderr.flush() +class WarningsPrinter: + def __init__(self, func: Callable[[str], None]) -> None: + self.func = func + # bpo-39983: Store the original sys.stderr at Python startup to be able to + # log warnings even if sys.stderr is captured temporarily by a test. + self.orig_stderr = sys.stderr + + def __call__(self, msg: str) -> None: + return self.func(msg) + + +@WarningsPrinter def print_warning(msg): # bpo-45410: Explicitly flush stdout to keep logs in order flush_std_streams() @@ -1333,10 +1346,6 @@ def print_warning(msg): print(f"Warning -- {line}", file=stream) stream.flush() -# bpo-39983: Store the original sys.stderr at Python startup to be able to -# log warnings even if sys.stderr is captured temporarily by a test. -print_warning.orig_stderr = sys.stderr - # Flag used by saved_test_environment of test.libregrtest.save_env, # to check if a test modified the environment. The flag should be set to False diff --git a/Tools/scripts/run_mypy_on_regrtest.py b/Tools/scripts/run_mypy_on_regrtest.py new file mode 100644 index 00000000000000..574c538fed3b26 --- /dev/null +++ b/Tools/scripts/run_mypy_on_regrtest.py @@ -0,0 +1,59 @@ +""" +Script to run mypy on Lib/test/libregrtest. + +This script is necessary due to the fact that, +if you invoke mypy directly on anything inside the Lib/ directory, +it (amusingly) thinks that everything in the stdlib is being "shadowed" +by the modules inside `Lib/`. +""" + +import argparse +import os +import tempfile +import shutil +import subprocess +from pathlib import Path +from typing import TypeAlias + +ReturnCode: TypeAlias = int + + +def run_mypy_on_libregrtest(stdlib_dir: Path) -> ReturnCode: + stdlib_test_dir = stdlib_dir / "test" + # Copy `Lib/test/support/` into a tempdir and point MYPYPATH towards the tempdir, + # so that mypy can see the classes and functions defined in `Lib/test/support/` + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + (td_path / "test").mkdir() + shutil.copytree(stdlib_test_dir / "support", td_path / "test" / "support") + mypy_command = [ + "mypy", + "--config-file", + "libregrtest/mypy.ini", + ] + result = subprocess.run( + mypy_command, cwd=stdlib_test_dir, env=os.environ | {"MYPYPATH": td} + ) + return result.returncode + + +def main() -> ReturnCode: + parser = argparse.ArgumentParser("Script to run mypy on Lib/test/regrtest/") + parser.add_argument( + "--stdlib-dir", + "-s", + type=Path, + required=True, + help="path to the Lib/ dir where the Python stdlib is located", + ) + args = parser.parse_args() + stdlib_dir = args.stdlib_dir + if not (stdlib_dir.exists() and stdlib_dir.is_dir()): + parser.error( + "--stdlib-dir must point to a directory that exists on your filesystem!" + ) + return run_mypy_on_libregrtest(args.stdlib_dir) + + +if __name__ == "__main__": + raise SystemExit(main()) From 18060746ee285fc0bcc11d11ae6c29a77e96c3c2 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 18 Sep 2023 13:19:49 +0100 Subject: [PATCH 2/2] Updates now #109518 is merged --- Lib/test/libregrtest/mypy.ini | 4 +++- Tools/scripts/run_mypy_on_regrtest.py | 29 +++++++++++++-------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Lib/test/libregrtest/mypy.ini b/Lib/test/libregrtest/mypy.ini index 8a0b89d480d477..722b32d778e252 100644 --- a/Lib/test/libregrtest/mypy.ini +++ b/Lib/test/libregrtest/mypy.ini @@ -31,4 +31,6 @@ strict_optional = False # Various internal modules that typeshed deliberately doesn't have stubs for: [mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] ignore_missing_imports = True -disable_error_code = attr-defined,var-annotated + +[mypy-test.support.*] +disable_error_code = attr-defined,var-annotated,index,assignment diff --git a/Tools/scripts/run_mypy_on_regrtest.py b/Tools/scripts/run_mypy_on_regrtest.py index 574c538fed3b26..772f366d8efc0d 100644 --- a/Tools/scripts/run_mypy_on_regrtest.py +++ b/Tools/scripts/run_mypy_on_regrtest.py @@ -9,30 +9,30 @@ import argparse import os -import tempfile import shutil import subprocess +import tempfile from pathlib import Path from typing import TypeAlias ReturnCode: TypeAlias = int -def run_mypy_on_libregrtest(stdlib_dir: Path) -> ReturnCode: - stdlib_test_dir = stdlib_dir / "test" +def run_mypy_on_libregrtest(root_dir: Path) -> ReturnCode: + test_dot_support_dir = root_dir / "Lib/" / "test" / "support" # Copy `Lib/test/support/` into a tempdir and point MYPYPATH towards the tempdir, # so that mypy can see the classes and functions defined in `Lib/test/support/` with tempfile.TemporaryDirectory() as td: td_path = Path(td) (td_path / "test").mkdir() - shutil.copytree(stdlib_test_dir / "support", td_path / "test" / "support") + shutil.copytree(test_dot_support_dir, td_path / "test" / "support") mypy_command = [ "mypy", "--config-file", - "libregrtest/mypy.ini", + "Lib/test/libregrtest/mypy.ini", ] result = subprocess.run( - mypy_command, cwd=stdlib_test_dir, env=os.environ | {"MYPYPATH": td} + mypy_command, cwd=root_dir, env=os.environ | {"MYPYPATH": td} ) return result.returncode @@ -40,19 +40,18 @@ def run_mypy_on_libregrtest(stdlib_dir: Path) -> ReturnCode: def main() -> ReturnCode: parser = argparse.ArgumentParser("Script to run mypy on Lib/test/regrtest/") parser.add_argument( - "--stdlib-dir", - "-s", + "--root-dir", + "-r", type=Path, required=True, - help="path to the Lib/ dir where the Python stdlib is located", + help="path to the CPython repo root", ) args = parser.parse_args() - stdlib_dir = args.stdlib_dir - if not (stdlib_dir.exists() and stdlib_dir.is_dir()): - parser.error( - "--stdlib-dir must point to a directory that exists on your filesystem!" - ) - return run_mypy_on_libregrtest(args.stdlib_dir) + root_dir = args.root_dir + test_dot_support_dir = root_dir / "Lib" / "test" / "support" + if not (test_dot_support_dir.exists() and test_dot_support_dir.is_dir()): + parser.error("--root-dir must point to the root of a CPython clone!") + return run_mypy_on_libregrtest(root_dir) if __name__ == "__main__":