chore(ci): update layer ARN docs and create PR during release

.github/actions/create-pr/create_pr_for_staged_changes.sh

@@ -3,7 +3,7 @@ set -uo pipefail # prevent accessing unset env vars, prevent masking pipeline er
 
 #docs
 #title              :create_pr_for_staged_changes.sh
-#description        :This script will create a PR for staged changes, detect and close duplicate PRs.
+#description        :This script will create a PR for staged changes, detect and close duplicate PRs. All PRs will be omitted from Release Notes and Changelogs
 #author		    :@heitorlessa
 #date               :May 8th 2023
 #version            :0.1
@@ -61,6 +61,8 @@ function set_environment_variables() {
     export readonly PR_BODY="This is an automated PR created from the following workflow"
     export readonly FILENAME=".github/scripts/$(basename "$0")"
     export readonly NO_DUPLICATES_MESSAGE="No duplicated PRs found"
+    export readonly SKIP_LABEL="skip-changelog"
+
     end_span
 }
 
@@ -86,7 +88,8 @@ function create_temporary_branch_with_changes() {
 
 function create_pr() {
     start_span "Creating PR against ${TEMP_BRANCH} branch"
-    NEW_PR_URL=$(gh pr create --title "${PR_TITLE}" --body "${PR_BODY}: ${WORKFLOW_URL}" --base "${BASE_BRANCH}" || error "Failed to create PR") # e.g, https://github.com/awslabs/aws-lambda-powertools/pull/13
+    # TODO: create label
+    NEW_PR_URL=$(gh pr create --title "${PR_TITLE}" --body "${PR_BODY}: ${WORKFLOW_URL}" --base "${BASE_BRANCH}" --label "${SKIP_LABEL}" || error "Failed to create PR") # e.g, https://github.com/awslabs/aws-lambda-powertools/pull/13
 
     # greedy remove any string until the last URL path, including the last '/'. https://opensource.com/article/17/6/bash-parameter-expansion
     debug "Extracing PR Number from PR URL: "${NEW_PR_URL}""

.github/workflows/publish_v2_layer.yml

@@ -1,10 +1,5 @@
 name: Deploy v2 layer to all regions
 
-permissions:
-  id-token: write
-  contents: write
-  pages: write
-
 on:
   workflow_dispatch:
     inputs:
@@ -31,7 +26,11 @@ on:
 jobs:
   build-layer:
     permissions:
+      # lower privilege propagated from parent workflow (release.yml)
       contents: read
+      id-token: none
+      pages: none
+      pull-requests: none
     runs-on: aws-lambda-powertools_ubuntu-latest_8-core
     defaults:
       run:
@@ -87,6 +86,12 @@ jobs:
 
   beta:
     needs: build-layer
+    # lower privilege propagated from parent workflow (release.yml)
+    permissions:
+      id-token: write
+      contents: read
+      pages: write             # docs will be updated with latest Layer ARNs
+      pull-requests: write # creation-action will create a PR with Layer ARN updates
     uses: ./.github/workflows/reusable_deploy_v2_layer_stack.yml
     secrets: inherit
     with:
@@ -97,6 +102,12 @@ jobs:
 
   prod:
     needs: beta
+    # lower privilege propagated from parent workflow (release.yml)
+    permissions:
+      id-token: write
+      contents: read
+      pages: write             # docs will be updated with latest Layer ARNs
+      pull-requests: write # creation-action will create a PR with Layer ARN updates
     uses: ./.github/workflows/reusable_deploy_v2_layer_stack.yml
     secrets: inherit
     with:
@@ -107,6 +118,12 @@ jobs:
 
   sar-beta:
     needs: build-layer
+    permissions:
+      # lower privilege propagated from parent workflow (release.yml)
+      id-token: write
+      contents: read
+      pull-requests: none
+      pages: none
     uses: ./.github/workflows/reusable_deploy_v2_sar.yml
     secrets: inherit
     with:
@@ -117,6 +134,12 @@ jobs:
 
   sar-prod:
     needs: [build-layer, sar-beta]
+    permissions:
+      # lower privilege propagated from parent workflow (release.yml)
+      id-token: write
+      contents: read
+      pull-requests: none
+      pages: none
     uses: ./.github/workflows/reusable_deploy_v2_sar.yml
     secrets: inherit
     with:
@@ -125,10 +148,62 @@ jobs:
       environment: "layer-prod"
       package-version: ${{ inputs.latest_published_version }}
 
+  # Updating the documentation with the latest Layer ARNs is a two-phase process
+  #
+  # 1. Update layer ARNs with latest deployed locally and create a PR with these changes
+  # 2. Pull from temporary branch with these changes and update the docs we're releasing
+  #
+  # This keeps our permissions tight and we don't run into a conflict,
+  # where a new release creates a new doc (2.16.0) while layers are still pointing to 2.15
+  # because the PR has to be merged while release process is running
+
+  update_v2_layer_arn_docs:
+    needs: prod
+    outputs:
+      temp_branch: ${{ steps.create-pr.outputs.temp_branch }}
+    runs-on: ubuntu-latest
+    permissions:
+      # lower privilege propagated from parent workflow (release.yml)
+      contents: write
+      pull-requests: write
+      id-token: none
+      pages: none
+    steps:
+      - name: Checkout repository # reusable workflows start clean, so we need to checkout again
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab  # v3.5.2
+        with:
+          fetch-depth: 0
+      - name: Download CDK layer artifact
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: cdk-layer-stack
+          path: cdk-layer-stack/
+      - name: Replace layer versions in documentation
+        run: |
+          ls -la cdk-layer-stack/
+          ./layer/scripts/update_layer_arn.sh cdk-layer-stack
+      # NOTE: It felt unnecessary creating yet another PR to update changelog w/ latest tag
+      # since this is the only step in the release where we update docs from a temp branch
+      - name: Update changelog with latest tag
+        run: make changelog
+      - name: Create PR
+        id: create-pr
+        uses: ./.github/actions/create-pr
+        with:
+          files: "docs/index.md examples CHANGELOG.md"
+          temp_branch_prefix: "ci-layer-docs"
+          pull_request_title: "chore(ci): layer docs update"
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+
   prepare_docs_alias:
     runs-on: ubuntu-latest
     permissions:
+      # lower privilege propagated from parent workflow (release.yml)
       contents: read
+      pages: none
+      id-token: none
+      pull-requests: none
     outputs:
       DOCS_ALIAS: ${{ steps.set-alias.outputs.DOCS_ALIAS }}
     steps:
@@ -141,13 +216,16 @@ jobs:
           fi
           echo DOCS_ALIAS="$DOCS_ALIAS" >> "$GITHUB_OUTPUT"
 
-  release-docs:
-    needs: [prod, prepare_docs_alias]
+  release_docs:
+    needs: [update_v2_layer_arn_docs, prepare_docs_alias]
     permissions:
+      # lower privilege propagated from parent workflow (release.yml)
       contents: write
       pages: write
+      pull-requests: none
+      id-token: none
     uses: ./.github/workflows/reusable_publish_docs.yml
     with:
       version: ${{ inputs.latest_published_version }}
       alias: ${{ needs.prepare_docs_alias.outputs.DOCS_ALIAS }}
-      detached_mode: true
+      git_ref: ${{ needs.update_v2_layer_arn_docs.outputs.temp_branch }}

.github/workflows/release.yml

@@ -5,12 +5,14 @@ name: Release
 # === Automated activities ===
 #
 # 1. Run tests, linting, security and complexity base line
-# 2. Bump package version, build release artifact, and generate latest Changelog
+# 2. Bump package version and build release artifact
 # 3. Publish package to PyPi prod repository using cached artifact
-# 4. Kick off Layers pipeline to compile and publish latest version
-# 5. Updates documentation to use the latest Layer ARN for all commercial regions
-# 6. Builds a new user guide and API docs with release version; update /latest pointing to newly released version
-# 7. Close all issues labeled "pending-release" and notify customers about the release
+# 4. Compile Layer and kick off pipeline for beta, prod, and canary releases
+# 5. Update docs with latest Layer ARNs and Changelog
+# 6. Create PR to update trunk so staged docs also point to the latest Layer ARN, when merged
+# 7. Builds a new user guide and API docs with release version; update /latest pointing to newly released version
+# 8. Create PR to update package version on trunk
+# 9. Close all issues labeled "pending-release" and notify customers about the release
 #
 # === Manual activities ===
 #
@@ -126,15 +128,36 @@ jobs:
       #   with:
       #     repository-url: https://test.pypi.org/legacy/
 
+  create_tag:
+    needs: [build, release]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      RELEASE_VERSION: ${{ needs.build.outputs.RELEASE_VERSION }}
+    steps:
+      - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab  # v3.5.2
+      - id: setup-git
+        name: Git client setup and refresh tip
+        run: |
+          git config user.name "Powertools bot"
+          git config user.email "[email protected]"
+          git config remote.origin.url >&-
+      - name: Create Git Tag
+        run: |
+          git tag -a v"${RELEASE_VERSION}" -m "release_version: v${RELEASE_VERSION}"
+          git push origin v"${RELEASE_VERSION}"
+
   # NOTE: Watch out for the depth limit of 4 nested workflow_calls.
-  # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack -> reusable_update_v2_layer_arn_docs
+  # publish_layer -> publish_v2_layer -> reusable_deploy_v2_layer_stack
   publish_layer:
-    needs: [build, release]
+    needs: [build, release, create_tag]
     secrets: inherit
     permissions:
       id-token: write
       contents: write
       pages: write
+      pull-requests: write
     uses: ./.github/workflows/publish_v2_layer.yml
     with:
       latest_published_version: ${{ needs.build.outputs.RELEASE_VERSION }}

.github/workflows/reusable_deploy_v2_layer_stack.yml

@@ -1,9 +1,5 @@
 name: Deploy CDK Layer v2 stack
 
-permissions:
-  id-token: write
-  contents: write
-
 on:
   workflow_call:
     inputs:
@@ -28,6 +24,12 @@ jobs:
   deploy-cdk-stack:
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
+    # lower privilege propagated from parent workflow (publish_v2_layer.yml)
+    permissions:
+      id-token: write
+      pull-requests: none
+      contents: read
+      pages: none
     defaults:
       run:
         working-directory: ./layer
@@ -149,10 +151,3 @@ jobs:
           retention-days: 1
       - name: CDK Deploy Canary
         run: npx cdk deploy --app cdk.out --context region=${{ matrix.region }} --parameters DeployStage="${{ inputs.stage }}" --parameters HasARM64Support=${{ matrix.has_arm64_support }} 'CanaryV2Stack' --require-approval never --verbose
-
-  update_v2_layer_arn_docs:
-    needs: deploy-cdk-stack
-    if: ${{ inputs.stage == 'PROD' }}
-    uses: ./.github/workflows/reusable_update_v2_layer_arn_docs.yml
-    with:
-      latest_published_version: ${{ inputs.latest_published_version }}

.github/workflows/reusable_publish_docs.yml

@@ -1,7 +1,6 @@
 name: Reusable publish documentation
 
 env:
-  BRANCH: develop
   ORIGIN: awslabs/aws-lambda-powertools-python
 
 on:
@@ -20,6 +19,11 @@ on:
         required: false
         default: false
         type: boolean
+      git_ref:
+        description: "Branch or commit ID to checkout from"
+        required: false
+        type: string
+        default: develop
 
 permissions:
   contents: write
@@ -36,6 +40,7 @@ jobs:
       - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab  # v3.5.2
         with:
           fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
       - name: Install poetry
         run: pipx install poetry
       - name: Set up Python
@@ -56,6 +61,8 @@ jobs:
           git config pull.rebase true
           git config remote.origin.url >&- || git remote add origin https://github.com/"$ORIGIN"
           git pull origin "$BRANCH"
+        env:
+          BRANCH: ${{ inputs.git_ref }}
       - name: Build docs website and API reference
         env:
           VERSION: ${{ inputs.version }}

layer/scripts/update_layer_arn.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# This script is run during the reusable_update_v2_layer_arn_docs CI job,
+# This script is run during the publish_v2_layer.yml CI job,
 # and it is responsible for replacing the layer ARN in our documentation,
 # based on the output files generated by CDK when deploying to each pseudo_region.
 #