diff --git a/stub_uploader/__init__.py b/stub_uploader/__init__.py index e69de29b..c5adf888 100644 --- a/stub_uploader/__init__.py +++ b/stub_uploader/__init__.py @@ -0,0 +1,3 @@ +import sys + +assert sys.version_info >= (3, 9) diff --git a/stub_uploader/get_version.py b/stub_uploader/get_version.py index 4d95d4eb..9e296b80 100644 --- a/stub_uploader/get_version.py +++ b/stub_uploader/get_version.py @@ -12,19 +12,17 @@ from __future__ import annotations -import argparse -import os.path -from collections.abc import Iterable -from typing import Any, Optional +from typing import Any, Union import requests -import tomli from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet from packaging.version import Version from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry from stub_uploader.const import * +from stub_uploader.metadata import Metadata PREFIX = "types-" URL_TEMPLATE = "https://pypi.org/pypi/{}/json" @@ -33,7 +31,7 @@ TIMEOUT = 3 -def fetch_pypi_versions(distribution: str) -> Iterable[str]: +def fetch_pypi_versions(distribution: str) -> list[Version]: url = URL_TEMPLATE.format(PREFIX + distribution) retry_strategy = Retry(total=RETRIES, status_forcelist=RETRY_ON) with requests.Session() as session: @@ -45,28 +43,7 @@ def fetch_pypi_versions(distribution: str) -> Iterable[str]: return [] raise ValueError("Error while retrieving version") releases: dict[str, Any] = resp.json()["releases"] - return releases.keys() - - -def read_base_version(typeshed_dir: str, distribution: str) -> str: - """Read distribution version from metadata.""" - metadata_file = os.path.join( - typeshed_dir, THIRD_PARTY_NAMESPACE, distribution, "METADATA.toml" - ) - with open(metadata_file, "rb") as f: - data = tomli.load(f) - version = data["version"] - assert isinstance(version, str) - if version.endswith(".*"): - version = version[:-2] - # Check that the version parses - Version(version) - return version - - -def strip_dep_version(dependency: str) -> str: - """Strip a possible version suffix, e.g. types-six>=0.1.4 -> types-six.""" - return Requirement(dependency).name + return [Version(release) for release in releases.keys()] def check_exists(distribution: str) -> bool: @@ -83,28 +60,87 @@ def check_exists(distribution: str) -> bool: raise ValueError("Error while verifying existence") -def main(typeshed_dir: str, distribution: str, version: Optional[str]) -> int: - """A simple function to get version increment of a third-party stub package. - - Supports basic reties and timeouts (as module constants). - """ - pypi_versions = fetch_pypi_versions(distribution) - if not version: - # Use the METADATA.toml version, if not given one. - version = read_base_version(typeshed_dir, distribution) - matching = [v for v in pypi_versions if v.startswith(f"{version}.")] - if not matching: - return -1 - increment = max(int(v.split(".")[-1]) for v in matching) - return increment - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("typeshed_dir", help="Path to typeshed checkout directory") - parser.add_argument("distribution", help="Third-party distribution to build") - parser.add_argument( - "version", nargs="?", help="Base version for which to get increment" +def ensure_specificity(ver: list[int], specificity: int) -> None: + ver.extend([0] * (specificity - len(ver))) + + +def compute_incremented_version( + version_spec: str, published_versions: list[Version] +) -> Version: + # The most important postcondition is that the incremented version is greater than + # all published versions. This ensures that users who don't pin get the most + # up to date stub. If we ever maintain multiple versions for a stub, this will + # need revisiting. + max_published = max(published_versions, default=Version("0")) + + # The second thing we try to do (but don't guarantee), is that the incremented + # version will satisfy the version_spec (defined precisely by the `compatible` + # specifier below). This allows users to have expectations of what a stub package + # will contain based on the upstream version they're targeting. + if version_spec.endswith(".*"): + compatible = SpecifierSet(f"=={version_spec}") + else: + compatible = SpecifierSet(f"=={version_spec}.*") + + # Look up the base version and specificity in METADATA.toml. + version_base = Version(version_spec.removesuffix(".*")) + specificity = len(version_base.release) + + if max_published.epoch > 0 or version_base.epoch > 0: + raise NotImplementedError("Epochs in versions are not supported") + + increment_specificity = specificity + 1 + # TODO: uncomment this in follow-up PR + # We'll try to bump the fourth part of the release. So e.g. if our version_spec is + # 1.1, we'll release 1.1.0.0, 1.1.0.1, 1.1.0.2, ... + # But if our version_spec is 5.6.7.8, we'll release 5.6.7.8.0, 5.6.7.8.1, ... + # increment_specificity = max(specificity + 1, 4) + + if version_base.release < max_published.release[:specificity]: + raise AssertionError("TODO: remove this exception in follow-up PR") + + # Our published versions have gone too far ahead the upstream version + # So we can't guarantee our second property. + # In practice, this will only happen if the specificity of version_spec is + # changed or we change our versioning scheme. + # For example, version_base=1.2, max_published=1.3.0.4, return 1.3.0.5 + increment_specificity = max(increment_specificity, len(max_published.release)) + incremented = list(max_published.release) + ensure_specificity(incremented, increment_specificity) + incremented[-1] += 1 + + incremented_version = Version(".".join(map(str, incremented))) + assert incremented_version > max_published + # But can't keep versioning compatible with upstream... + assert incremented_version not in compatible + return incremented_version + + if version_base.release > max_published.release[:specificity]: + # For example, version_base=1.2, max_published=1.1.0.4, return 1.2.0.0 + incremented = list(version_base.release) + ensure_specificity(incremented, increment_specificity) + + else: + assert version_base.release == max_published.release[:specificity] + # For example, version_base=1.1, max_published=1.1.0.4, return 1.1.0.5 + incremented = list(max_published.release) + ensure_specificity(incremented, increment_specificity) + incremented[-1] += 1 + + incremented_version = Version(".".join(map(str, incremented))) + assert incremented_version > max_published + assert incremented_version in compatible + return incremented_version + + +def strip_dep_version(dependency: str) -> str: + """Strip a possible version suffix, e.g. types-six>=0.1.4 -> types-six.""" + return Requirement(dependency).name + + +def determine_incremented_version(metadata: Metadata) -> str: + published_stub_versions = fetch_pypi_versions(metadata.distribution) + version = compute_incremented_version( + metadata.version_spec, published_stub_versions ) - args = parser.parse_args() - print(main(args.typeshed_dir, args.distribution, args.version)) + return str(version) diff --git a/stub_uploader/metadata.py b/stub_uploader/metadata.py index 326d2105..30423fb0 100644 --- a/stub_uploader/metadata.py +++ b/stub_uploader/metadata.py @@ -3,19 +3,21 @@ import tomli -from stub_uploader import get_version - from .const import META, THIRD_PARTY_NAMESPACE class Metadata: - def __init__(self, data: Dict[str, Any]): + def __init__(self, distribution: str, data: Dict[str, Any]): + self.distribution = distribution self.data = data - @classmethod - def from_file(cls, path: str) -> "Metadata": - with open(path, "rb") as f: - return cls(tomli.load(f)) + @property + def version_spec(self) -> str: + # The "version" field in METADATA.toml isn't actually a version, it's more + # like a specifier, e.g. we allow it to contain wildcards. + version = self.data["version"] + assert isinstance(version, str) + return version @property def requires(self) -> List[str]: @@ -37,13 +39,6 @@ def no_longer_updated(self) -> bool: def read_metadata(typeshed_dir: str, distribution: str) -> Metadata: """Parse metadata from file.""" path = os.path.join(typeshed_dir, THIRD_PARTY_NAMESPACE, distribution, META) - return Metadata.from_file(path) - - -def determine_version(typeshed_dir: str, distribution: str) -> str: - version = get_version.read_base_version(typeshed_dir, distribution) - increment = get_version.main(typeshed_dir, distribution, version) - if increment >= 0: - print(f"Existing version found for {distribution}") - increment += 1 - return f"{version}.{increment}" + with open(path, "rb") as f: + data = tomli.load(f) + return Metadata(distribution=distribution, data=data) diff --git a/stub_uploader/upload_changed.py b/stub_uploader/upload_changed.py index 2445cea6..0d77c43a 100644 --- a/stub_uploader/upload_changed.py +++ b/stub_uploader/upload_changed.py @@ -14,7 +14,8 @@ import subprocess from stub_uploader import build_wheel, get_changed, update_changelog -from stub_uploader.metadata import determine_version, read_metadata +from stub_uploader.metadata import read_metadata +from stub_uploader.get_version import determine_incremented_version def main(typeshed_dir: str, commit: str, uploaded: str, dry_run: bool = False) -> None: @@ -29,7 +30,8 @@ def main(typeshed_dir: str, commit: str, uploaded: str, dry_run: bool = False) - ) print("Building and uploading stubs for:", ", ".join(to_upload)) for distribution in to_upload: - version = determine_version(typeshed_dir, distribution) + metadata = read_metadata(typeshed_dir, distribution) + version = determine_incremented_version(metadata) update_changelog.update_changelog( typeshed_dir, commit, distribution, version, dry_run=dry_run ) @@ -37,7 +39,7 @@ def main(typeshed_dir: str, commit: str, uploaded: str, dry_run: bool = False) - if dry_run: print(f"Would upload: {distribution}, version {version}") continue - for dependency in read_metadata(typeshed_dir, distribution).requires: + for dependency in metadata.requires: build_wheel.verify_dependency(typeshed_dir, dependency, uploaded) subprocess.run(["twine", "upload", os.path.join(temp_dir, "*")], check=True) build_wheel.update_uploaded(uploaded, distribution) diff --git a/stub_uploader/upload_some.py b/stub_uploader/upload_some.py index d70d2b55..e5315f0f 100644 --- a/stub_uploader/upload_some.py +++ b/stub_uploader/upload_some.py @@ -14,7 +14,8 @@ import subprocess from stub_uploader import build_wheel -from stub_uploader.metadata import determine_version, read_metadata +from stub_uploader.metadata import read_metadata +from stub_uploader.get_version import determine_incremented_version def main(typeshed_dir: str, pattern: str, uploaded: str) -> None: @@ -31,8 +32,9 @@ def main(typeshed_dir: str, pattern: str, uploaded: str) -> None: ) print("Uploading stubs for:", ", ".join(to_upload)) for distribution in to_upload: - version = determine_version(typeshed_dir, distribution) - for dependency in read_metadata(typeshed_dir, distribution).requires: + metadata = read_metadata(typeshed_dir, distribution) + version = determine_incremented_version(metadata) + for dependency in metadata.requires: build_wheel.verify_dependency(typeshed_dir, dependency, uploaded) # TODO: Update changelog temp_dir = build_wheel.main(typeshed_dir, distribution, version) diff --git a/tests/test_integration.py b/tests/test_integration.py index ec0ea9d8..4e297ab7 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -5,6 +5,7 @@ """ import os import pytest +from packaging.version import Version from stub_uploader import get_version, build_wheel from stub_uploader.metadata import read_metadata @@ -12,12 +13,11 @@ UPLOADED = "data/uploaded_packages.txt" -def test_version() -> None: +def test_fetch_pypi_versions() -> None: """Check that we can query PyPI for package increments.""" - assert get_version.main(TYPESHED, "six", "0.1") >= 0 - assert get_version.main(TYPESHED, "nonexistent-distribution", "0.1") == -1 - assert get_version.main(TYPESHED, "typed-ast", "0.1") == -1 - assert get_version.main(TYPESHED, "typed-ast", None) >= 0 + assert Version("1.16.0") in get_version.fetch_pypi_versions("six") + assert Version("1.5.4") in get_version.fetch_pypi_versions("typed-ast") + assert not get_version.fetch_pypi_versions("nonexistent-distribution") def test_check_exists() -> None: @@ -33,6 +33,11 @@ def test_build_wheel(distribution: str) -> None: assert list(os.listdir(tmp_dir)) # check it is not empty +@pytest.mark.parametrize("distribution", os.listdir(os.path.join(TYPESHED, "stubs"))) +def test_version_increment(distribution: str) -> None: + get_version.determine_incremented_version(read_metadata(TYPESHED, distribution)) + + def test_verify_dependency() -> None: # Check some known dependencies that they verify as valid. build_wheel.verify_dependency(TYPESHED, "types-six", UPLOADED) diff --git a/tests/test_unit.py b/tests/test_unit.py index f5630643..3fb8497f 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -2,13 +2,17 @@ import os import pytest -from stub_uploader.get_version import strip_dep_version +from packaging.version import Version +from stub_uploader.get_version import ( + compute_incremented_version, + ensure_specificity, + strip_dep_version, +) from stub_uploader.build_wheel import ( collect_setup_entries, sort_by_dependency, transitive_deps, strip_types_prefix, - BuildData, ) @@ -27,6 +31,83 @@ def test_strip_version() -> None: assert strip_dep_version("types-foo>2.3") == "types-foo" +def test_ensure_specificity() -> None: + ver = [1] + ensure_specificity(ver, 3) + assert ver == [1, 0, 0] + + ver = [1, 2, 3] + ensure_specificity(ver, 3) + assert ver == [1, 2, 3] + + ver = [1, 2, 3, 4, 5] + ensure_specificity(ver, 3) + assert ver == [1, 2, 3, 4, 5] + + +def _incremented_ver(ver: str, published: list[str]) -> str: + published_vers = [Version(v) for v in published] + return str(compute_incremented_version(ver, published_vers)) + + +def test_compute_incremented_version_legacy() -> None: + # never before published version + empty_list: list[str] = [] + assert _incremented_ver("1", empty_list) == "1.0" + assert _incremented_ver("1.2", empty_list) == "1.2.0" + + # published less than version spec + assert _incremented_ver("1.2", ["1.1.0.4"]) == "1.2.0" + assert _incremented_ver("1", ["0.9"]) == "1.0" + assert _incremented_ver("1.1", ["0.9"]) == "1.1.0" + assert _incremented_ver("1.2.3", ["1.1.0.17"]) == "1.2.3.0" + assert _incremented_ver("1.2.3.4", ["1.1.0.17"]) == "1.2.3.4.0" + + # published equals version spec + assert _incremented_ver("1.1", ["1.1"]) == "1.1.1" + assert _incremented_ver("1.1", ["1.1.0.4"]) == "1.1.0.5" + assert _incremented_ver("1.1", ["1.1.3.4"]) == "1.1.3.5" + assert _incremented_ver("1.2.3.4", ["1.2.3.4.5"]) == "1.2.3.4.6" + assert _incremented_ver("1.2.3.4.5", ["1.2.3.4.5"]) == "1.2.3.4.5.1" + + # test that we do the max version right + assert _incremented_ver("1.2", ["1.1.0.7", "1.2.0.7"]) == "1.2.0.8" + + +@pytest.mark.skip(reason="Will use in follow-up PR") +def test_compute_incremented_version() -> None: + # never before published version + empty_list: list[str] = [] + assert _incremented_ver("1", empty_list) == "1.0.0.0" + assert _incremented_ver("1.2", empty_list) == "1.2.0.0" + + # published greater than version spec + assert _incremented_ver("1.2", ["1.3.0.4"]) == "1.3.0.5" + assert _incremented_ver("1.1", ["1.2.0.1"]) == "1.2.0.2" + assert _incremented_ver("1.1", ["1.2"]) == "1.2.0.1" + assert _incremented_ver("1.1", ["1.2.3"]) == "1.2.3.1" + assert _incremented_ver("1.1", ["1.2.3.4.5"]) == "1.2.3.4.6" + assert _incremented_ver("1.4.40", ["1.4.50"]) == "1.4.50.1" + assert _incremented_ver("1.4.0.40", ["1.4.0.50"]) == "1.4.0.50.1" + + # published less than version spec + assert _incremented_ver("1.2", ["1.1.0.4"]) == "1.2.0.0" + assert _incremented_ver("1", ["0.9"]) == "1.0.0.0" + assert _incremented_ver("1.1", ["0.9"]) == "1.1.0.0" + assert _incremented_ver("1.2.3", ["1.1.0.17"]) == "1.2.3.0" + assert _incremented_ver("1.2.3.4", ["1.1.0.17"]) == "1.2.3.4.0" + + # published equals version spec + assert _incremented_ver("1.1", ["1.1"]) == "1.1.0.1" + assert _incremented_ver("1.1", ["1.1.0.4"]) == "1.1.0.5" + assert _incremented_ver("1.1", ["1.1.3.4"]) == "1.1.3.5" + assert _incremented_ver("1.2.3.4", ["1.2.3.4.5"]) == "1.2.3.4.6" + assert _incremented_ver("1.2.3.4.5", ["1.2.3.4.5"]) == "1.2.3.4.5.1" + + # test that we do the max version right + assert _incremented_ver("1.2", ["1.1.0.7", "1.2.0.7", "1.3.0.7"]) == "1.3.0.8" + + def test_transitive_deps() -> None: with pytest.raises(KeyError): # We require the graph to be complete for safety.