Extract VERSIONS parsing to _utils.py (#12351)

Author: srittau

tests/_utils.py

@@ -7,7 +7,8 @@
 from collections.abc import Iterable, Mapping
 from functools import lru_cache
 from pathlib import Path
-from typing import Any, Final, NamedTuple
+from typing import Any, Final, NamedTuple, Tuple
+from typing_extensions import TypeAlias
 
 import pathspec
 from packaging.requirements import Requirement
@@ -106,8 +107,35 @@ def get_mypy_req() -> str:
 # Parsing the stdlib/VERSIONS file
 # ====================================================================
 
+VersionTuple: TypeAlias = Tuple[int, int]
 
-VERSIONS_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d{1,2})?$")
+
+VERSIONS_PATH = STDLIB_PATH / "VERSIONS"
+VERSION_LINE_RE = re.compile(r"^([a-zA-Z_][a-zA-Z0-9_.]*): ([23]\.\d{1,2})-([23]\.\d{1,2})?$")
+VERSION_RE = re.compile(r"^([23])\.(\d+)$")
+
+
+def parse_stdlib_versions_file() -> dict[str, tuple[VersionTuple, VersionTuple]]:
+    result: dict[str, tuple[VersionTuple, VersionTuple]] = {}
+    with VERSIONS_PATH.open(encoding="UTF-8") as f:
+        for line in f:
+            line = strip_comments(line)
+            if line == "":
+                continue
+            m = VERSION_LINE_RE.match(line)
+            assert m, f"invalid VERSIONS line: {line}"
+            mod: str = m.group(1)
+            assert mod not in result, f"Duplicate module {mod} in VERSIONS"
+            min_version = _parse_version(m.group(2))
+            max_version = _parse_version(m.group(3)) if m.group(3) else (99, 99)
+            result[mod] = min_version, max_version
+    return result
+
+
+def _parse_version(v_str: str) -> tuple[int, int]:
+    m = VERSION_RE.match(v_str)
+    assert m, f"invalid version: {v_str}"
+    return int(m.group(1)), int(m.group(2))
 
 
 # ====================================================================

tests/check_typeshed_structure.py

@@ -18,12 +18,11 @@
     STDLIB_PATH,
     TEST_CASES_DIR,
     TESTS_DIR,
-    VERSIONS_RE,
     get_all_testcase_directories,
     get_gitignore_spec,
     parse_requirements,
+    parse_stdlib_versions_file,
     spec_matches_path,
-    strip_comments,
     tests_path,
 )
 
@@ -127,30 +126,17 @@ def check_no_symlinks() -> None:
 
 def check_versions_file() -> None:
     """Check that the stdlib/VERSIONS file has the correct format."""
-    versions = list[str]()
-    with open("stdlib/VERSIONS", encoding="UTF-8") as f:
-        data = f.read().splitlines()
-    for line in data:
-        line = strip_comments(line)
-        if line == "":
-            continue
-        m = VERSIONS_RE.match(line)
-        if not m:
-            raise AssertionError(f"Bad line in VERSIONS: {line}")
-        module = m.group(1)
-        assert module not in versions, f"Duplicate module {module} in VERSIONS"
-        versions.append(module)
-
-    deduped_versions = set(versions)
-    assert len(versions) == len(deduped_versions)
+    version_map = parse_stdlib_versions_file()
+    versions = list(version_map.keys())
+
     sorted_versions = sorted(versions)
     assert versions == sorted_versions, f"{versions=}\n\n{sorted_versions=}"
 
     modules = _find_stdlib_modules()
     # Sub-modules don't need to be listed in VERSIONS.
-    extra = {m.split(".")[0] for m in modules} - deduped_versions
+    extra = {m.split(".")[0] for m in modules} - version_map.keys()
     assert not extra, f"Modules not in versions: {extra}"
-    extra = deduped_versions - modules
+    extra = version_map.keys() - modules
     assert not extra, f"Versions not in modules: {extra}"
 
 

tests/mypy_test.py

@@ -6,7 +6,6 @@
 import argparse
 import concurrent.futures
 import os
-import re
 import subprocess
 import sys
 import tempfile
@@ -17,11 +16,7 @@
 from itertools import product
 from pathlib import Path
 from threading import Lock
-from typing import TYPE_CHECKING, Any, NamedTuple, Tuple
-
-if TYPE_CHECKING:
-    from _typeshed import StrPath
-
+from typing import Any, NamedTuple
 from typing_extensions import Annotated, TypeAlias
 
 import tomli
@@ -30,14 +25,13 @@
 from _utils import (
     PYTHON_VERSION,
     TESTS_DIR,
-    VERSIONS_RE as VERSION_LINE_RE,
     colored,
     get_gitignore_spec,
     get_mypy_req,
+    parse_stdlib_versions_file,
     print_error,
     print_success_msg,
     spec_matches_path,
-    strip_comments,
     venv_python,
 )
 
@@ -53,7 +47,6 @@
 DIRECTORIES_TO_TEST = [Path("stdlib"), Path("stubs")]
 
 VersionString: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_VERSIONS"]
-VersionTuple: TypeAlias = Tuple[int, int]
 Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"]
 
 
@@ -150,31 +143,6 @@ def match(path: Path, args: TestConfig) -> bool:
     return False
 
 
-def parse_versions(fname: StrPath) -> dict[str, tuple[VersionTuple, VersionTuple]]:
-    result: dict[str, tuple[VersionTuple, VersionTuple]] = {}
-    with open(fname, encoding="UTF-8") as f:
-        for line in f:
-            line = strip_comments(line)
-            if line == "":
-                continue
-            m = VERSION_LINE_RE.match(line)
-            assert m, f"invalid VERSIONS line: {line}"
-            mod: str = m.group(1)
-            min_version = parse_version(m.group(2))
-            max_version = parse_version(m.group(3)) if m.group(3) else (99, 99)
-            result[mod] = min_version, max_version
-    return result
-
-
-_VERSION_RE = re.compile(r"^([23])\.(\d+)$")
-
-
-def parse_version(v_str: str) -> tuple[int, int]:
-    m = _VERSION_RE.match(v_str)
-    assert m, f"invalid version: {v_str}"
-    return int(m.group(1)), int(m.group(2))
-
-
 def add_files(files: list[Path], module: Path, args: TestConfig) -> None:
     """Add all files in package or module represented by 'name' located in 'root'."""
     if module.is_file() and module.suffix == ".pyi":
@@ -365,7 +333,7 @@ def test_third_party_distribution(
 def test_stdlib(args: TestConfig) -> TestResult:
     files: list[Path] = []
     stdlib = Path("stdlib")
-    supported_versions = parse_versions(stdlib / "VERSIONS")
+    supported_versions = parse_stdlib_versions_file()
     for name in os.listdir(stdlib):
         if name in ("VERSIONS", TESTS_DIR) or name.startswith("."):
             continue