From c407883a1b46b26a46520852f8f4d96b1bd8eedd Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 13 Jul 2022 16:21:03 +0200 Subject: [PATCH 1/7] chore(ci): remove version lock on changelog generation --- .chglog/CHANGELOG.tpl.md | 16 ++++++++++------ .chglog/config.yml | 3 +++ Makefile | 5 +++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.chglog/CHANGELOG.tpl.md b/.chglog/CHANGELOG.tpl.md index b1fa7de1f58..b8919554b80 100755 --- a/.chglog/CHANGELOG.tpl.md +++ b/.chglog/CHANGELOG.tpl.md @@ -1,10 +1,13 @@ + + + {{ if .Versions -}} -## Unreleased +# Unreleased {{ if .Unreleased.CommitGroups -}} {{ range .Unreleased.CommitGroups -}} -### {{ .Title }} +## {{ .Title }} {{ range .Commits -}} * {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} @@ -17,7 +20,8 @@ ## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }} {{ range .CommitGroups -}} -### {{ .Title }} + +## {{ .Title }} {{ range .Commits -}} * {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} @@ -25,14 +29,14 @@ {{ end -}} {{- if .RevertCommits -}} -### Reverts +## Reverts {{ range .RevertCommits -}} * {{ .Revert.Header }} {{ end }} {{ end -}} {{- if .MergeCommits -}} -### Pull Requests +## Pull Requests {{ range .MergeCommits -}} * {{ .Header }} @@ -41,7 +45,7 @@ {{- if .NoteGroups -}} {{ range .NoteGroups -}} -### {{ .Title }} +## {{ .Title }} {{ range .Notes }} {{ .Body }} {{ end }} diff --git a/.chglog/config.yml b/.chglog/config.yml index 3392563d445..21577651526 100755 --- a/.chglog/config.yml +++ b/.chglog/config.yml @@ -32,3 +32,6 @@ options: notes: keywords: - BREAKING CHANGE + # issues: + # prefix: + # - # diff --git a/Makefile b/Makefile index 6173e3e310d..bf001caa5f2 100644 --- a/Makefile +++ b/Makefile @@ -91,8 +91,9 @@ release: pr changelog: git fetch --tags origin - @echo "[+] Pre-generating CHANGELOG for tag: $$(git describe --abbrev=0 --tag)" - docker run -v "${PWD}":/workdir quay.io/git-chglog/git-chglog $$(git describe --abbrev=0 --tag).. > TMP_CHANGELOG.md + CURRENT_VERSION=$(shell git describe --abbrev=0 --tag) ;\ + echo "[+] Pre-generating CHANGELOG for tag: $$CURRENT_VERSION" ;\ + docker run -v "${PWD}":/workdir quay.io/git-chglog/git-chglog > CHANGELOG.md mypy: poetry run mypy --pretty aws_lambda_powertools From ea123822eb1d08b478beda15dbeab327cbde5e95 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 14 Jul 2022 09:11:16 +0200 Subject: [PATCH 2/7] chore(ci): bump version and changelog upon release Signed-off-by: heitorlessa --- .github/workflows/publish.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c3b6537f314..293e7b833c2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -54,14 +54,24 @@ jobs: run: | RELEASE_TAG_VERSION=${{ github.event.release.tag_name }} echo "RELEASE_TAG_VERSION=${RELEASE_TAG_VERSION:1}" >> $GITHUB_ENV - - name: Ensure new version is also set in pyproject and CHANGELOG - run: | - grep --regexp "${RELEASE_TAG_VERSION}" CHANGELOG.md - grep --regexp "version \= \"${RELEASE_TAG_VERSION}\"" pyproject.toml - name: Install dependencies run: make dev - name: Run all tests, linting and baselines run: make pr + - name: Bump package version + run: poetry version ${RELEASE_TAG_VERSION} + - name: Generate latest CHANGELOG + run: make changelog + - name: Setup git client + run: | + git config user.name "Release bot" + git config user.email aws-devax-open-source@amazon.com + - name: Push project metadata and changelog to trunk + run: | + git add CHANGELOG.md + git add pyproject.toml + git commit -m "chore(ci): update project with version ${RELEASE_TAG_VERSION}" + git push origin HEAD:refs/heads/develop - name: Build python package and wheel run: poetry build - name: Upload to PyPi test @@ -84,10 +94,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: eu-west-1 AWS_DEFAULT_OUTPUT: json - - name: Setup doc deploy - run: | - git config --global user.name Docs deploy - git config --global user.email aws-devax-open-source@amazon.com - name: Build docs website and API reference run: | make release-docs VERSION=${RELEASE_TAG_VERSION} ALIAS="latest" From 47b2505767c0253c03a27e95cb861e73ddcaad60 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 14 Jul 2022 09:20:17 +0200 Subject: [PATCH 3/7] chore(ci): prevent conflict with changelog update upon release Signed-off-by: heitorlessa --- .github/workflows/python_docs.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_docs.yml b/.github/workflows/python_docs.yml index d7ae6c2cc52..e3c30e8c23e 100644 --- a/.github/workflows/python_docs.yml +++ b/.github/workflows/python_docs.yml @@ -1,12 +1,14 @@ name: Docs +# Maintenance: Create a reusable workflow to be more easily reused across release, push, and doc hot fixes +# this should include inputs on whether to release API docs, what version to release, and whether to rebuild /latest + on: push: branches: - develop paths: - "docs/**" - - "CHANGELOG.md" - "mkdocs.yml" - "examples/**" From 75eb680a1e0e65ce0ea44f1b151a4826e426a68e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 14 Jul 2022 09:35:02 +0200 Subject: [PATCH 4/7] chore(ci): split release job to ease maintenance --- .github/workflows/publish.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 293e7b833c2..fcc8c0a152e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -42,6 +42,8 @@ on: jobs: release: runs-on: ubuntu-latest + outputs: + RELEASE_TAG_VERSION: ${{ steps.release_version.outputs.RELEASE_TAG_VERSION }} steps: - uses: actions/checkout@v3 with: @@ -51,9 +53,11 @@ jobs: with: python-version: "3.8" - name: Set release notes tag + id: release_version run: | RELEASE_TAG_VERSION=${{ github.event.release.tag_name }} echo "RELEASE_TAG_VERSION=${RELEASE_TAG_VERSION:1}" >> $GITHUB_ENV + echo "::set-output name=RELEASE_TAG_VERSION::${RELEASE_TAG_VERSION}" - name: Install dependencies run: make dev - name: Run all tests, linting and baselines @@ -94,6 +98,16 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: eu-west-1 AWS_DEFAULT_OUTPUT: json + + docs: + needs: release + runs-on: ubuntu-latest + env: + RELEASE_TAG_VERSION: ${{ needs.release.outputs.RELEASE_TAG_VERSION }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Build docs website and API reference run: | make release-docs VERSION=${RELEASE_TAG_VERSION} ALIAS="latest" @@ -112,9 +126,18 @@ jobs: publish_dir: ./api keep_files: true destination_dir: latest/api + + post_release: + needs: release + runs-on: ubuntu-latest + env: + RELEASE_TAG_VERSION: ${{ needs.release.outputs.RELEASE_TAG_VERSION }} + steps: + - uses: actions/checkout@v3 - name: Close issues related to this release uses: actions/github-script@v6 with: + github-token: ${{ secrets.GITHUB_TOKEN }} script: | const post_release = require('.github/scripts/post_release.js') await post_release({github, context, core}) From bc4da145527807f31d3881610fcc14c57d2dd9fb Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 14 Jul 2022 10:22:46 +0200 Subject: [PATCH 5/7] chore(governance): update release steps Signed-off-by: heitorlessa --- .github/workflows/publish.yml | 34 +++++++++------------------------- MAINTAINERS.md | 34 ++++++++++++++++------------------ 2 files changed, 25 insertions(+), 43 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index fcc8c0a152e..8fd528955fd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,36 +4,20 @@ name: Publish to PyPi # # === Manual activities === # -# 1. Document human readable changes in CHANGELOG (pre-generate unreleased changes with `make changelog`) -# 2. Bump package version using poetry version -# 3. Merge version changes to develop branch -# 4. Edit the current draft release notes -# 5. If not already set, use `v` as a tag, and select develop as target branch +# 1. Edit the current draft release notes +# 2. If not already set, use `v` as a tag, e.g., v1.26.4, and select develop as target branch # # === Automated activities === # # 1. Extract release notes tag that was published -# 2. Ensure release notes tag match what's in CHANGELOG and pyproject -# 3. Run tests, linting, security and complexity base line -# 4. Publish package to PyPi test repository -# 5. Publish package to PyPi prod repository -# 6. Kick off Lambda Layer pipeline to publish latest version with minimal dependencies as a SAR App -# 7. Kick off Lambda Layer pipeline to publish latest version with extra dependencies as a SAR App -# 8. Builds a fresh version of docs including Changelog updates -# 9. Builds latest documentation for new release, and update latest alias pointing to the new release tag -# 10. Close and notify all issues labeled "pending-release" about the release details +# 2. Run tests, linting, security and complexity base line +# 3. Bump package version and generate latest Changelog +# 4. Publish package to PyPi test and prod repository +# 5. Kick off SAR App pipeline to publish latest version with minimal and extra dependencies +# 6. Builds a new user guide and API docs with release version; update /latest pointing to newly released version +# 7. Close all issues labeled "pending-release" and notify customers about the release -# -# === Fallback mechanism due to external failures === -# -# 1. Trigger "Publish to PyPi" workflow manually: https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow -# 2. Use the version released under Releases e.g. v1.13.0 -# - -# -# === Documentation hotfix === -# -# Look for rebuild latest docs workflow +# See MAINTAINERS.md "Releasing a new version" for release mechanisms on: release: diff --git a/MAINTAINERS.md b/MAINTAINERS.md index fa8b3287238..3f8d812823a 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -169,23 +169,15 @@ Some examples using our initial and new RFC templates: #92, #94, #95, #991, #122 ### Releasing a new version -> TODO: This is an area we want to increase automation while keeping communication at human level. +Firstly, make sure the commit history in the `develop` branch **(1)** it's up to date, **(2)** commit messages are semantic, and **(3)** commit messages have their respective area, for example `feat(logger): `, `chore(ci): ...`). -Firstly, make sure you are using the `develop` branch and it is up to date with the origin. +**Found typos or unclear commit messages?** -There are three main steps to release a new version: Changelog generation, version bumping, and drafting release notes. +Reword through rebase and push with `--force-with-lease` once you're confident. This will ensure [CHANGELOG](./CHANGELOG.md) is always clear for customers looking to understand what changed in between releases - was that a bug? what new features and for which utility? -#### Changelog generation +**Looks good, what's next?** -You can pre-generate a temporary CHANGELOG using `make changelog`. This will generate a `TMP_CHANGELOG.md` with all staged changes under the `unreleased` section. - -Each unreleased line item is a commit. You can adjust them if you find the commit titles are insufficient to describe their intent. Once you're comfortable, bring these changes to the `CHANGELOG.md` with a new version heading like in previous versions. - -#### Bumping the version - -Use `poetry version ` to bump the version. For example, you can use `poetry version minor` when releasing a minor version. - -NOTE. Make sure both `CHANGELOG` and `pyproject.toml` are committed and pushed to the remote `develop` branch before proceeding. +The only step is to draft and publish a good release notes, everything else is automated. #### Drafting release notes @@ -195,21 +187,27 @@ Make sure the `tag` field reflects the new version you're releasing, the target You'll notice we group all changes based on their [labels](#labels) like `feature`, `bug`, `documentation`, etc. -> **Q: What if there's an incorrect title or grouping?** +**I spotted a typo or incorrect grouping - how do I fix it?** Edit the respective PR title and update their [labels](#labels). Then run the [Release Drafter workflow](https://github.com/awslabs/aws-lambda-powertools-python/actions/workflows/release-drafter.yml) to update the Draft release. -The best part comes now. Replace the placeholder `[Human readable summary of changes]` with what you'd like to communicate to customers what this release is all about. Always put yourself in the customers shoes. For that, these are some questions to keep in mind when drafting your first or future release notes: +**All looking good, what's next?** + +The best part comes now. Replace the placeholder `[Human readable summary of changes]` with what you'd like to communicate to customers what this release is all about. Rule of thumb: always put yourself in the customers shoes. + +These are some questions to keep in mind when drafting your first or future release notes: - Can customers understand at a high level what changed in this release? - Is there a link to the documentation where they can read more about each main change? -- Are there any graphics or code snippets that can enhance readability? +- Are there any graphics or [code snippets](carbon.now.sh/) that can enhance readability? - Are we calling out any key contributor(s) to this release? - All contributors are automatically credited, use this as an exceptional case to feature them -Once you're happy, hit `Publish release`. This will kick off the [Publishing workflow](https://github.com/awslabs/aws-lambda-powertools-python/actions/workflows/publish.yml) and within a few minutes you should see the latest version in PyPi, and all issues labeled as `pending-release` will be notified. +Once you're happy, hit `Publish release` 🎉🎉🎉. + +This will kick off the [Publishing workflow](https://github.com/awslabs/aws-lambda-powertools-python/actions/workflows/publish.yml) and within a few minutes you should see the latest version in PyPi, and all issues labeled as `pending-release` will be closed and notified. -> TODO: Wait for @am29d new Lambda Layers pipeline work to complete, then add how Lambda Layers are published +> TODO: Include information to verify SAR and Lambda Layers deployment; we're still finalizing Lambda Layer automated deployment in GitHub Actions - ping @am29d when in doubt. ### Releasing a documentation hotfix From 37726c4d987ca0c7369876a107436820430f64e6 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 14 Jul 2022 10:45:55 +0200 Subject: [PATCH 6/7] chore(ci): support manual trigger if gh action has failures --- .github/scripts/post_release.js | 6 +++--- .github/workflows/publish.yml | 31 ++++++++++++++++++++----------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/scripts/post_release.js b/.github/scripts/post_release.js index 70474740e2e..d6a598f1960 100644 --- a/.github/scripts/post_release.js +++ b/.github/scripts/post_release.js @@ -100,13 +100,13 @@ const notifyRelease = async ({ // context: https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts module.exports = async ({ github, context }) => { - const { RELEASE_TAG_VERSION } = process.env; - console.log(`Running post-release script for ${RELEASE_TAG_VERSION} version`); + const { RELEASE_VERSION } = process.env; + console.log(`Running post-release script for ${RELEASE_VERSION} version`); await notifyRelease({ gh_client: github, owner: context.repo.owner, repository: context.repo.repo, - release_version: RELEASE_TAG_VERSION, + release_version: RELEASE_VERSION, }); }; diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8fd528955fd..236356748fd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,12 +22,20 @@ name: Publish to PyPi on: release: types: [published] + workflow_dispatch: + inputs: + version_to_publish: + description: "Version to be released in PyPi, Docs, and Lambda Layer, e.g. v1.26.4" + default: v1.26.4 + required: true jobs: release: runs-on: ubuntu-latest outputs: - RELEASE_TAG_VERSION: ${{ steps.release_version.outputs.RELEASE_TAG_VERSION }} + RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }} + env: + RELEASE_TAG_VERSION: ${{ github.event.release.tag_name || inputs.version_to_publish }} steps: - uses: actions/checkout@v3 with: @@ -38,16 +46,17 @@ jobs: python-version: "3.8" - name: Set release notes tag id: release_version + # transform tag format `v> $GITHUB_ENV - echo "::set-output name=RELEASE_TAG_VERSION::${RELEASE_TAG_VERSION}" + RELEASE_VERSION=${RELEASE_TAG_VERSION:1} + echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV + echo "::set-output name=RELEASE_VERSION::${RELEASE_VERSION}" - name: Install dependencies run: make dev - name: Run all tests, linting and baselines run: make pr - name: Bump package version - run: poetry version ${RELEASE_TAG_VERSION} + run: poetry version ${RELEASE_VERSION} - name: Generate latest CHANGELOG run: make changelog - name: Setup git client @@ -58,7 +67,7 @@ jobs: run: | git add CHANGELOG.md git add pyproject.toml - git commit -m "chore(ci): update project with version ${RELEASE_TAG_VERSION}" + git commit -m "chore(ci): update project with version ${RELEASE_VERSION}" git push origin HEAD:refs/heads/develop - name: Build python package and wheel run: poetry build @@ -74,7 +83,7 @@ jobs: PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - name: publish lambda layer in SAR by triggering the internal codepipeline run: | - aws ssm put-parameter --name "powertools-python-release-version" --value $RELEASE_TAG_VERSION --overwrite + aws ssm put-parameter --name "powertools-python-release-version" --value $RELEASE_VERSION --overwrite aws codepipeline start-pipeline-execution --name ${{ secrets.CODEPIPELINE_NAME }} env: # Maintenance: Migrate to new OAuth mechanism @@ -87,14 +96,14 @@ jobs: needs: release runs-on: ubuntu-latest env: - RELEASE_TAG_VERSION: ${{ needs.release.outputs.RELEASE_TAG_VERSION }} + RELEASE_VERSION: ${{ needs.release.outputs.RELEASE_VERSION }} steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Build docs website and API reference run: | - make release-docs VERSION=${RELEASE_TAG_VERSION} ALIAS="latest" + make release-docs VERSION=${RELEASE_VERSION} ALIAS="latest" poetry run mike set-default --push latest - name: Release API docs to release version uses: peaceiris/actions-gh-pages@v3 @@ -102,7 +111,7 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./api keep_files: true - destination_dir: ${{ env.RELEASE_TAG_VERSION }}/api + destination_dir: ${{ env.RELEASE_VERSION }}/api - name: Release API docs to latest uses: peaceiris/actions-gh-pages@v3 with: @@ -115,7 +124,7 @@ jobs: needs: release runs-on: ubuntu-latest env: - RELEASE_TAG_VERSION: ${{ needs.release.outputs.RELEASE_TAG_VERSION }} + RELEASE_VERSION: ${{ needs.release.outputs.RELEASE_VERSION }} steps: - uses: actions/checkout@v3 - name: Close issues related to this release From 4330cab13f2cfcd8b16d1bf704042341c8d5765c Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Thu, 14 Jul 2022 10:48:36 +0200 Subject: [PATCH 7/7] chore(ci): ensure git client is configured for docs release --- .github/workflows/publish.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 236356748fd..1668be72f1c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -101,6 +101,10 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Setup git client + run: | + git config user.name "Release bot" + git config user.email aws-devax-open-source@amazon.com - name: Build docs website and API reference run: | make release-docs VERSION=${RELEASE_VERSION} ALIAS="latest"