From 141607c425c9cbd2f691fbe0db65d64f366ae898 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 9 Oct 2023 13:25:17 +0100 Subject: [PATCH 1/6] Add changelog generation script --- misc/generate_changelog.py | 158 +++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 misc/generate_changelog.py diff --git a/misc/generate_changelog.py b/misc/generate_changelog.py new file mode 100644 index 000000000000..20443d1946a7 --- /dev/null +++ b/misc/generate_changelog.py @@ -0,0 +1,158 @@ +"""Generate the changelog for a mypy release.""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from dataclasses import dataclass + + +def find_all_release_branches() -> None: + result = subprocess.run(["git", "branch", "-r"], text=True, capture_output=True, check=True) + versions = [] + for line in result.stdout.splitlines(): + line = line.strip() + if m := re.match(r"origin/release-([0-9]+)\.([0-9]+)$", line): + major = int(m.group(1)) + minor = int(m.group(2)) + versions.append((major, minor)) + return versions + + +def git_merge_base(rev1: str, rev2: str) -> str: + result = subprocess.run( + ["git", "merge-base", rev1, rev2], text=True, capture_output=True, check=True + ) + return result.stdout.strip() + + +@dataclass +class CommitInfo: + commit: str + author: str + title: str + pr_number: int | None + + +def normalize_author(author: str) -> str: + # Some ad-hoc rules to get more consistent author names. + if author == "AlexWaygood": + return "Alex Waygood" + return author + + +def git_commit_log(rev1: str, rev2: str) -> list[CommitInfo]: + result = subprocess.run( + ["git", "log", "--pretty=%H\t%an\t%s", f"{rev1}..{rev2}"], + text=True, + capture_output=True, + check=True, + ) + commits = [] + for line in result.stdout.splitlines(): + commit, author, title = line.strip().split("\t", 2) + pr_number = None + if m := re.match(r".*\(#([0-9]+)\)$", title): + pr_number = int(m.group(1)) + title = re.sub(r" *\(#[0-9]+\)$", "", title) + + author = normalize_author(author) + entry = CommitInfo(commit, author, title, pr_number) + commits.append(entry) + return commits + + +def filter_omitted_commits(commits: list[CommitInfo]) -> list[CommitInfo]: + result = [] + for c in commits: + title = c.title + keep = True + if title.startswith("Sync typeshed"): + # Typeshed syncs aren't mentioned in release notes + keep = False + if title in ( + "Revert sum literal integer change", + "Remove use of LiteralString in builtins", + "Revert typeshed ctypes change", + "Revert use of `ParamSpec` for `functools.wraps`", + ): + # These are generated by a typeshed sync. + keep = False + if re.search(r"(bump|update).*version.*\+dev", title.lower()): + # Version number updates aren't mentioned + keep = False + if "pre-commit autoupdate" in title: + keep = False + if title.startswith("Update commit hashes"): + # Internal tool change + keep = False + if keep: + result.append(c) + return result + + +def find_changes_between_releases(old_branch: str, new_branch: str) -> list[str]: + merge_base = git_merge_base(old_branch, new_branch) + print(f"Merge base: {merge_base}") + new_commits = git_commit_log(merge_base, new_branch) + old_commits = git_commit_log(merge_base, new_branch) + + # Filter out some commits that won't be mentioned in release notes. + new_commits = filter_omitted_commits(new_commits) + + # Filter out commits cherry-picked to old branch. + # TODO + + return new_commits + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("release") + parser.add_argument("--local", action="store_true") + args = parser.parse_args() + release: str = args.release + local: bool = args.local + + if not re.match(r"[0-9]+\.[0-9]+$", release): + sys.exit(f"error: Release must be of form X.Y (not {release!r})") + major, minor = [int(component) for component in release.split(".")] + + if not local: + print("Running 'git fetch' to fetch all release branches...") + subprocess.run(["git", "fetch"], check=True) + + if minor > 0: + prev_major = major + prev_minor = minor - 1 + else: + # For a x.0 release, the previous release is the most recent (x-1).y release. + all_releases = sorted(find_all_release_branches()) + if (major, minor) not in all_releases: + sys.exit(f"error: Can't find release branch for {major}.{minor} at origin") + for i in reversed(range(len(all_releases))): + if all_releases[i][0] == major - 1: + prev_major, prev_minor = all_releases[i] + break + else: + sys.exit("error: Could not determine previous release") + print(f"Generating changelog for {major}.{minor}") + print(f"Previous release was {prev_major}.{prev_minor}") + + new_branch = f"origin/release-{major}.{minor}" + old_branch = f"origin/release-{prev_major}.{prev_minor}" + + changes = find_changes_between_releases(old_branch, new_branch) + + print() + for c in changes: + s = f" * {c.title} ({c.author})" + if c.pr_number: + s += f" (#{c.pr_number})" + print(s) + + +if __name__ == "__main__": + main() From 37abea6afe8f487f4a57d726f33534e5d6a0fcc6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 9 Oct 2023 14:02:16 +0100 Subject: [PATCH 2/6] Drop commits included in previous release plus a few other fixes --- misc/generate_changelog.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/misc/generate_changelog.py b/misc/generate_changelog.py index 20443d1946a7..b36c995b3026 100644 --- a/misc/generate_changelog.py +++ b/misc/generate_changelog.py @@ -54,9 +54,9 @@ def git_commit_log(rev1: str, rev2: str) -> list[CommitInfo]: for line in result.stdout.splitlines(): commit, author, title = line.strip().split("\t", 2) pr_number = None - if m := re.match(r".*\(#([0-9]+)\)$", title): + if m := re.match(r".*\(#([0-9]+)\) *$", title): pr_number = int(m.group(1)) - title = re.sub(r" *\(#[0-9]+\)$", "", title) + title = re.sub(r" *\(#[0-9]+\) *$", "", title) author = normalize_author(author) entry = CommitInfo(commit, author, title, pr_number) @@ -85,7 +85,7 @@ def filter_omitted_commits(commits: list[CommitInfo]) -> list[CommitInfo]: keep = False if "pre-commit autoupdate" in title: keep = False - if title.startswith("Update commit hashes"): + if title.startswith(("Update commit hashes", "Update hashes")): # Internal tool change keep = False if keep: @@ -93,17 +93,43 @@ def filter_omitted_commits(commits: list[CommitInfo]) -> list[CommitInfo]: return result +def normalize_title(title: str) -> str: + # We sometimes add a title prefix when cherry-picking commits to a + # release branch. Attempt to remove these prefixes so that we can + # match them to the corresponding master branch. + if m := re.match(r"\[release [0-9.]+\] *", title, flags=re.I): + title = title.replace(m.group(0), "") + return title + + +def filter_out_commits_from_old_release_branch(new_commits: list[CommitInfo], + old_commits: list[CommitInfo]) -> list[CommitInfo]: + old_titles = {normalize_title(commit.title) for commit in old_commits} + result = [] + for commit in new_commits: + drop = False + if normalize_title(commit.title) in old_titles: + drop = True + if normalize_title(f"{commit.title} (#{commit.pr_number})") in old_titles: + drop = True + if not drop: + result.append(commit) + else: + print(f'NOTE: Drop "{commit.title}", since it was in previous release branch') + return result + + def find_changes_between_releases(old_branch: str, new_branch: str) -> list[str]: merge_base = git_merge_base(old_branch, new_branch) print(f"Merge base: {merge_base}") new_commits = git_commit_log(merge_base, new_branch) - old_commits = git_commit_log(merge_base, new_branch) + old_commits = git_commit_log(merge_base, old_branch) # Filter out some commits that won't be mentioned in release notes. new_commits = filter_omitted_commits(new_commits) # Filter out commits cherry-picked to old branch. - # TODO + new_commits = filter_out_commits_from_old_release_branch(new_commits, old_commits) return new_commits From 38ce36cf5ecbb86c66ad71ec69127ff0f0323ca5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 9 Oct 2023 16:05:45 +0100 Subject: [PATCH 3/6] Fixes --- misc/generate_changelog.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misc/generate_changelog.py b/misc/generate_changelog.py index b36c995b3026..e5425381baed 100644 --- a/misc/generate_changelog.py +++ b/misc/generate_changelog.py @@ -72,12 +72,12 @@ def filter_omitted_commits(commits: list[CommitInfo]) -> list[CommitInfo]: if title.startswith("Sync typeshed"): # Typeshed syncs aren't mentioned in release notes keep = False - if title in ( + if title.startswith(( "Revert sum literal integer change", "Remove use of LiteralString in builtins", "Revert typeshed ctypes change", "Revert use of `ParamSpec` for `functools.wraps`", - ): + )): # These are generated by a typeshed sync. keep = False if re.search(r"(bump|update).*version.*\+dev", title.lower()): @@ -174,9 +174,10 @@ def main() -> None: print() for c in changes: - s = f" * {c.title} ({c.author})" + s = f" * {c.commit[:9]} - {c.title}" if c.pr_number: s += f" (#{c.pr_number})" + s += f" ({c.author})" print(s) From c32d20d6881b6de21dfef783867f5cd664591fd3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 Nov 2023 11:29:34 +0000 Subject: [PATCH 4/6] Switch to current changelog format in the repo --- misc/generate_changelog.py | 50 ++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/misc/generate_changelog.py b/misc/generate_changelog.py index e5425381baed..a3da052b3e31 100644 --- a/misc/generate_changelog.py +++ b/misc/generate_changelog.py @@ -72,12 +72,14 @@ def filter_omitted_commits(commits: list[CommitInfo]) -> list[CommitInfo]: if title.startswith("Sync typeshed"): # Typeshed syncs aren't mentioned in release notes keep = False - if title.startswith(( - "Revert sum literal integer change", - "Remove use of LiteralString in builtins", - "Revert typeshed ctypes change", - "Revert use of `ParamSpec` for `functools.wraps`", - )): + if title.startswith( + ( + "Revert sum literal integer change", + "Remove use of LiteralString in builtins", + "Revert typeshed ctypes change", + "Revert use of `ParamSpec` for `functools.wraps`", + ) + ): # These are generated by a typeshed sync. keep = False if re.search(r"(bump|update).*version.*\+dev", title.lower()): @@ -102,8 +104,9 @@ def normalize_title(title: str) -> str: return title -def filter_out_commits_from_old_release_branch(new_commits: list[CommitInfo], - old_commits: list[CommitInfo]) -> list[CommitInfo]: +def filter_out_commits_from_old_release_branch( + new_commits: list[CommitInfo], old_commits: list[CommitInfo] +) -> list[CommitInfo]: old_titles = {normalize_title(commit.title) for commit in old_commits} result = [] for commit in new_commits: @@ -134,17 +137,32 @@ def find_changes_between_releases(old_branch: str, new_branch: str) -> list[str] return new_commits +def format_changelog_entry(c: CommitInfo) -> str: + """ + s = f" * {c.commit[:9]} - {c.title}" + if c.pr_number: + s += f" (#{c.pr_number})" + s += f" ({c.author})" + """ + s = f" * {c.title} ({c.author}" + if c.pr_number: + s += f", PR [{c.pr_number}](https://github.com/python/mypy/pull/{c.pr_number})" + s += ")" + + return s + + def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("release") + parser.add_argument("version", help="target mypy version (form X.Y)") parser.add_argument("--local", action="store_true") args = parser.parse_args() - release: str = args.release + version: str = args.version local: bool = args.local - if not re.match(r"[0-9]+\.[0-9]+$", release): - sys.exit(f"error: Release must be of form X.Y (not {release!r})") - major, minor = [int(component) for component in release.split(".")] + if not re.match(r"[0-9]+\.[0-9]+$", version): + sys.exit(f"error: Release must be of form X.Y (not {version!r})") + major, minor = (int(component) for component in version.split(".")) if not local: print("Running 'git fetch' to fetch all release branches...") @@ -174,11 +192,7 @@ def main() -> None: print() for c in changes: - s = f" * {c.commit[:9]} - {c.title}" - if c.pr_number: - s += f" (#{c.pr_number})" - s += f" ({c.author})" - print(s) + print(format_changelog_entry(c)) if __name__ == "__main__": From 79405373490b2fd29207bf4c4f13d2edc5e5cc33 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 Nov 2023 11:38:00 +0000 Subject: [PATCH 5/6] Add author normalization special case for jhance --- misc/generate_changelog.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc/generate_changelog.py b/misc/generate_changelog.py index a3da052b3e31..1b620acb9b5f 100644 --- a/misc/generate_changelog.py +++ b/misc/generate_changelog.py @@ -40,6 +40,8 @@ def normalize_author(author: str) -> str: # Some ad-hoc rules to get more consistent author names. if author == "AlexWaygood": return "Alex Waygood" + elif author == "jhance": + return "Jared Hance" return author From fe04a5f32dac871d60495752a5ec0dee7e46d325 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 9 Nov 2023 11:39:45 +0000 Subject: [PATCH 6/6] Fix type checking --- misc/generate_changelog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/generate_changelog.py b/misc/generate_changelog.py index 1b620acb9b5f..7c7f28b6eeb7 100644 --- a/misc/generate_changelog.py +++ b/misc/generate_changelog.py @@ -9,7 +9,7 @@ from dataclasses import dataclass -def find_all_release_branches() -> None: +def find_all_release_branches() -> list[tuple[int, int]]: result = subprocess.run(["git", "branch", "-r"], text=True, capture_output=True, check=True) versions = [] for line in result.stdout.splitlines(): @@ -124,7 +124,7 @@ def filter_out_commits_from_old_release_branch( return result -def find_changes_between_releases(old_branch: str, new_branch: str) -> list[str]: +def find_changes_between_releases(old_branch: str, new_branch: str) -> list[CommitInfo]: merge_base = git_merge_base(old_branch, new_branch) print(f"Merge base: {merge_base}") new_commits = git_commit_log(merge_base, new_branch)