From 67e3e152c5821b597a011279a934abe1b94c4914 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 22 Mar 2022 15:53:59 +0100 Subject: [PATCH 1/7] Add script to list third-party package status --- scripts/third-party-status.py | 209 ++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100755 scripts/third-party-status.py diff --git a/scripts/third-party-status.py b/scripts/third-party-status.py new file mode 100755 index 000000000000..0c5d90fbc197 --- /dev/null +++ b/scripts/third-party-status.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 + +""" +Print information about third-party package versions. + +By default, this prints the latest stub version and the latest version +and release date on PyPI to the console. Optionally, it prints the +status as an HTML page. +""" + +from __future__ import annotations + +import asyncio +import datetime +from argparse import ArgumentParser +from asyncio.tasks import Task +from dataclasses import dataclass +from enum import Enum +from html import escape as html_escape +from pathlib import Path +from typing import Iterable, Iterator +from urllib.parse import quote as quote_url + +import aiohttp +import tomli + +try: + from termcolor import colored +except ImportError: + + def colored(s: str, c: str) -> str: + return s + + +NOW = datetime.datetime.utcnow() +OLD_YEARS = 2 +ANCIENT_YEARS = 5 + + +class OutputFormat(Enum): + TEXT = "text" + HTML = "html" + + +@dataclass +class StubInfo: + distribution: str + stub_version: str + pypi_version: str + pypi_date: datetime.datetime + + @property + def version_freshness(self) -> VersionFreshness: + if self.stub_version.endswith(".*"): + stub_parts = self.stub_version.split(".")[:-1] + pypi_parts = self.pypi_version.split(".") + assert len(stub_parts) <= len(pypi_parts) + if stub_parts == pypi_parts[: len(stub_parts)]: + return VersionFreshness.FRESH + elif stub_parts[0] == pypi_parts[0]: + return VersionFreshness.NEW_MINOR + else: + return VersionFreshness.NEW_MAJOR + else: + if self.stub_version == self.pypi_version: + return VersionFreshness.FRESH + else: + return VersionFreshness.NEW_MINOR + + @property + def date_freshness(self) -> DateFreshness: + days = (NOW - self.pypi_date).days + if days > 365 * ANCIENT_YEARS: + return DateFreshness.ANCIENT + elif days > 265 * OLD_YEARS: + return DateFreshness.OLD + else: + return DateFreshness.FRESH + + +class VersionFreshness(Enum): + FRESH = "fresh" + NEW_MINOR = "new-minor" + NEW_MAJOR = "new-major" + + +class DateFreshness(Enum): + FRESH = "fresh" + OLD = "old" + ANCIENT = "ancient" + + +VERSION_FRESHNESS_COLORS = { + VersionFreshness.FRESH: "green", + VersionFreshness.NEW_MINOR: "yellow", + VersionFreshness.NEW_MAJOR: "red", +} + +DATE_FRESHNESS_COLORS = {DateFreshness.FRESH: "green", DateFreshness.OLD: "yellow", DateFreshness.ANCIENT: "red"} + + +def main() -> None: + format = parse_args() + ts_stubs = list(read_typeshed_stubs()) + pypi_stubs = asyncio.run(fetch_pypi_stubs(t[0] for t in ts_stubs)) + stubs = [ + StubInfo(distribution, stub_version, pypi_version, pypi_date) + for (distribution, stub_version), (pypi_version, pypi_date) in zip(ts_stubs, pypi_stubs) + ] + stubs.sort(key=lambda si: si.distribution) + if format == OutputFormat.HTML: + print_stubs_html(stubs) + else: + print_stubs_text(stubs) + + +def parse_args() -> OutputFormat: + formats = [o.value for o in OutputFormat] + parser = ArgumentParser() + parser.add_argument("--output-format", type=str, choices=formats, default=formats[0]) + args = parser.parse_args() + return OutputFormat(args.output_format) + + +def read_typeshed_stubs() -> Iterator[tuple[str, str]]: + """Yield (distribution, version) tuples for all third-party stubs in typeshed.""" + for stub_path in Path("stubs").iterdir(): + with (stub_path / "METADATA.toml").open("rb") as f: + meta = tomli.load(f) + yield stub_path.name, meta["version"] + + +async def fetch_pypi_stubs(distributions: Iterable[str]) -> tuple[tuple[str, datetime.datetime], ...]: + async with aiohttp.ClientSession() as session: + tasks: list[Task[tuple[str, datetime.datetime]]] = [] + for distribution in distributions: + tasks.append(asyncio.create_task(fetch_pypi_info(session, distribution))) + return await asyncio.gather(*tasks) + + +async def fetch_pypi_info(session: aiohttp.ClientSession, distribution: str) -> tuple[str, datetime.datetime]: + """Return the latest version and release date of a distribution on PyPI.""" + url = f"https://pypi.org/pypi/{quote_url(distribution)}/json" + async with session.get(url) as response: + assert response.status == 200 + j = await response.json() + version = j["info"]["version"] + date = datetime.datetime.fromisoformat(j["releases"][version][0]["upload_time"]) + return version, date + + +def print_stubs_text(stubs: Iterable[StubInfo]) -> None: + dist_len = max(len(st.distribution) for st in stubs) + 2 + stub_v_len = max(len(st.stub_version) for st in stubs) + 2 + pypi_v_len = max(len(st.pypi_version) for st in stubs) + 2 + for stub in stubs: + version_color = VERSION_FRESHNESS_COLORS[stub.version_freshness] + date_color = DATE_FRESHNESS_COLORS[stub.date_freshness] + print(stub.distribution, end="") + print(" " * (dist_len - len(stub.distribution)), end="") + print(colored(stub.stub_version, version_color), end="") + print(" " * (stub_v_len - len(stub.stub_version)), end="") + print(stub.pypi_version, end="") + print(" " * (pypi_v_len - len(stub.pypi_version)), end="") + print(colored(stub.pypi_date.date().isoformat(), date_color)) + + +HTML_HEADER = f""" + + + typeshed Third-Party Stub Status + + + +

Last update: {NOW.date().isoformat()}

+ + + + + +""" + +HTML_FOOTER = "
DistributionStubPyPILast Release
" + + +def print_stubs_html(stubs: Iterable[StubInfo]) -> None: + print(HTML_HEADER) + for stub in stubs: + print( + f""" + + {html_escape(stub.distribution)} + {html_escape(stub.stub_version)} + {html_escape(stub.pypi_version)} + {stub.pypi_date.date().isoformat()} + + """ + ) + print(HTML_FOOTER) + + +if __name__ == "__main__": + main() From b56172b5ba4fae9bff6f20a1640bbeba5835dd0d Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 22 Mar 2022 16:04:39 +0100 Subject: [PATCH 2/7] Be more friendly to pypi.org --- scripts/third-party-status.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/third-party-status.py b/scripts/third-party-status.py index 0c5d90fbc197..adfb83593a32 100755 --- a/scripts/third-party-status.py +++ b/scripts/third-party-status.py @@ -131,7 +131,8 @@ def read_typeshed_stubs() -> Iterator[tuple[str, str]]: async def fetch_pypi_stubs(distributions: Iterable[str]) -> tuple[tuple[str, datetime.datetime], ...]: - async with aiohttp.ClientSession() as session: + conn = aiohttp.TCPConnector(limit_per_host=10) + async with aiohttp.ClientSession(connector=conn) as session: tasks: list[Task[tuple[str, datetime.datetime]]] = [] for distribution in distributions: tasks.append(asyncio.create_task(fetch_pypi_info(session, distribution))) From 644721b83e2f1d7213e73e85247ae8fa30e7ddfa Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 22 Mar 2022 16:07:08 +0100 Subject: [PATCH 3/7] Fix trailing whitespace --- scripts/third-party-status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/third-party-status.py b/scripts/third-party-status.py index adfb83593a32..21565a1af6f4 100755 --- a/scripts/third-party-status.py +++ b/scripts/third-party-status.py @@ -170,7 +170,7 @@ def print_stubs_text(stubs: Iterable[StubInfo]) -> None: typeshed Third-Party Stub Status - @@ -196,7 +200,7 @@ def print_stubs_html(stubs: Iterable[StubInfo]) -> None: print( f""" - {html_escape(stub.distribution)} + {html_escape(stub.distribution)} {html_escape(stub.stub_version)} {html_escape(stub.pypi_version)} {stub.pypi_date.date().isoformat()} From fbbc33198dd7ea73c80517cc060cc0eedb6ad28a Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 22 Mar 2022 16:44:30 +0100 Subject: [PATCH 5/7] Don't color output when not connected to a TTY --- scripts/third-party-status.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/third-party-status.py b/scripts/third-party-status.py index 434e1cd99f5c..779395273cad 100755 --- a/scripts/third-party-status.py +++ b/scripts/third-party-status.py @@ -12,6 +12,7 @@ import asyncio import datetime +import sys from argparse import ArgumentParser from asyncio.tasks import Task from dataclasses import dataclass @@ -157,16 +158,23 @@ def print_stubs_text(stubs: Iterable[StubInfo]) -> None: stub_v_len = max(len(st.stub_version) for st in stubs) + 2 pypi_v_len = max(len(st.pypi_version) for st in stubs) + 2 for stub in stubs: - distribution_text = colored(stub.distribution, "red") if stub.obsolete else stub.distribution + distribution_text = color(stub.distribution, "red") if stub.obsolete else stub.distribution version_color = VERSION_FRESHNESS_COLORS[stub.version_freshness] date_color = DATE_FRESHNESS_COLORS[stub.date_freshness] print(distribution_text, end="") print(" " * (dist_len - len(stub.distribution)), end="") - print(colored(stub.stub_version, version_color), end="") + print(color(stub.stub_version, version_color), end="") print(" " * (stub_v_len - len(stub.stub_version)), end="") print(stub.pypi_version, end="") print(" " * (pypi_v_len - len(stub.pypi_version)), end="") - print(colored(stub.pypi_date.date().isoformat(), date_color)) + print(color(stub.pypi_date.date().isoformat(), date_color)) + + +def color(s: str, color: str) -> str: + if sys.stdout.isatty(): + return colored(s, color) + else: + return s HTML_HEADER = f""" From c7ef6ef745ac3ee8270e2448e21c2e98d8a55327 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 22 Mar 2022 17:54:01 +0100 Subject: [PATCH 6/7] Try to fix GitHub From 5cc8798f8d4570669d3a6d9a2e75959347de8d13 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 7 Jun 2022 18:27:16 -0700 Subject: [PATCH 7/7] Update scripts/third-party-status.py --- scripts/third-party-status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/third-party-status.py b/scripts/third-party-status.py index 779395273cad..902bf7ff260c 100755 --- a/scripts/third-party-status.py +++ b/scripts/third-party-status.py @@ -74,7 +74,7 @@ def date_freshness(self) -> DateFreshness: days = (NOW - self.pypi_date).days if days > 365 * ANCIENT_YEARS: return DateFreshness.ANCIENT - elif days > 265 * OLD_YEARS: + elif days > 365 * OLD_YEARS: return DateFreshness.OLD else: return DateFreshness.FRESH