feat: optimize skip_push and skip_retag (#233)

* feat: optimize skip_push and skip_retag

* Disable trivy cache

* Use correct components

* Disable trivy for now

* Skip empty matrix builds

* Remove some more things for testing

* Only provide test_images_report if folder isn't empty

Author: puehringer

.github/workflows/build-docker-artifacts.yml

@@ -102,23 +102,37 @@ jobs:
 
             const builds = process.env.BUILDS ? process.env.BUILDS.split(',') : Object.keys(config.build);
             const push_to = process.env.PUSH_TO ? process.env.PUSH_TO.split(',') : Object.keys(config.push || {});
+            const skip_push = process.env.SKIP_PUSH === 'true';
 
             const flavors = builds.map(id => [id, config.build[id]]).filter(([id, flavor]) => flavor.skip !== true).map(([id, flavor]) => {
+              const hooksDirectory = path.join('./deploy/build', flavor.directory, 'hooks');
+              const testImagesHookScript = path.join(hooksDirectory, 'test-images.sh');
+              const testImagesHookReport = path.join(hooksDirectory, 'test-images-report');
+              const testImageEnabled = fs.existsSync(testImagesHookScript);
+
               return {
                 ...flavor,
                 id,
+                skip_retag: skip_push,
                 // Add metadata to the flavor object (will be used as matrix input)
                 build_time: buildTime,
                 image_tag: imageTag,
                 image_tag_branch_name: imageTagBranchName,
-                ecr_respositories: flavor.components.map(component => component.ecr_repository),
+                ecr_repositories: flavor.components.map(component => component.ecr_repository),
+                test_images: {
+                  enabled: testImageEnabled,
+                  script_path: testImagesHookScript,
+                  report_path: testImagesHookReport,
+                },
                 components: flavor.components.map(component => {
                   // Format build arguments as build-args string
                   const formattedBuildArgs = component.build_args ?
                     Object.entries(component.build_args).map(([key, value]) => `${key}=${value}`).join('\n') : '';
 
                   return {
                     ...component,
+                    // We can skip the push only if we don't want to test the image below
+                    skip_push: skip_push && !testImageEnabled,
                     // Add metadata to the component object (will be used as matrix input),
                     flavor,
                     flavor_id: id,
@@ -133,16 +147,18 @@ jobs:
               };
             });
 
-            const flattenedComponents = flavors.flatMap(flavor => flavor.components);
-
             const result = {
               flavors,
-              components: flattenedComponents,
+              flavorsToTest: flavors.filter(flavor => flavor.test_images.enabled),
+              flavorsToRetag: flavors.filter(flavor => flavor.skip_retag !== true),
+              componentsToBuild: flavors.flatMap(flavor => flavor.components),
               push_to: push_to.join(','),
+              skip_push,
             };
             console.log(result);
             return result;
         env:
+          SKIP_PUSH: ${{ inputs.skip_push }}
           BUILDS: ${{ inputs.builds }}
           PUSH_TO: ${{ inputs.push_to }}
 
@@ -152,7 +168,7 @@ jobs:
     strategy:
       fail-fast: ${{ inputs.fail_fast }}
       matrix:
-        component: ${{ fromJson(needs.get-flavors.outputs.result).components }}
+        component: ${{ fromJson(needs.get-flavors.outputs.result).componentsToBuild }}
     runs-on: ${{ vars.BUILD_DOCKER_RUNS_ON_OVERRIDE || vars.RUNS_ON_OVERRIDE || inputs.runs_on || 'ubuntu-22.04' }}
     steps:
       - name: View flavor and component
@@ -279,9 +295,12 @@ jobs:
           ignore-unfixed: false
           vuln-type: "os,library"
           severity: ${{ steps.set_severity.outputs.severity }}
+          # The cache update takes quite long, so let's try to disable it for now: https://github.com/aquasecurity/trivy-action#cache
+          cache: "false"
         continue-on-error: false
 
       - name: Push image
+        if: ${{ matrix.component.skip_push != true }}
         # Instead of the docker/build-push-action@v6 which will rebuild the image, just push it directly
         run: docker push ${{ matrix.component.image_ref }}
 
@@ -292,10 +311,12 @@ jobs:
   test-images:
     name: Test images of flavor ${{ matrix.flavor.id || 'default' }}
     needs: [get-flavors, build-flavors]
+    # Skip in case we would have an empty matrix. Note that in this case it also skips all downstream jobs...
+    if: ${{ toJSON(fromJson(needs.get-flavors.outputs.result).flavorsToTest) != '[]' }}
     strategy:
       fail-fast: false
       matrix:
-        flavor: ${{ fromJson(needs.get-flavors.outputs.result).flavors }}
+        flavor: ${{ fromJson(needs.get-flavors.outputs.result).flavorsToTest }}
     runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }}
     steps:
       - name: Checkout repository
@@ -304,13 +325,6 @@ jobs:
           ref: ${{ inputs.branch || github.sha }}
           token: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
 
-      - name: Checkout github-workflows repository
-        uses: actions/checkout@v5
-        with:
-          repository: datavisyn/github-workflows
-          ref: ${{ env.WORKFLOW_BRANCH }}
-          path: ./tmp/github-workflows
-
       - name: Configure AWS Credentials
         uses: aws-actions/[email protected]
         with:
@@ -325,9 +339,8 @@ jobs:
         shell: bash
         id: test-images
         run: |
-          hooks_folder="$(realpath -m "./deploy/build/${{ matrix.flavor.directory }}/hooks")"
-          test_images_hook="$hooks_folder/test-images.sh"
-          test_images_report="$hooks_folder/test-images-report"
+          test_images_hook=$(jq -r '.test_images.script_path' <<< "$FLAVOR")
+          test_images_report=$(jq -r '.test_images.report_path' <<< "$FLAVOR")
 
           if [[ -f "$test_images_hook" ]]; then
             # Iterate through all components and store their image ref in an environment variable
@@ -340,15 +353,17 @@ jobs:
               export "IMAGE_${name_upper}=${image_ref}"
             done;
 
-            # Create report folder to avoid any downstream Docker volume issues
-            # TODO: For some reason this doesn't work yet, i.e. if a docker-compose script mounts a volume here, nothing shows up...
+            # Create a directory to store the report results
             mkdir -p "$test_images_report"
-            chmod 777 "$test_images_report"
-            echo "test_images_report=${test_images_report}" >> "$GITHUB_OUTPUT"
 
             echo "Run $test_images_hook"
             chmod +x "$test_images_hook"
             bash "$test_images_hook"
+
+            # If the report folder is not empty, we store it as output
+            if [[ -n "$(ls -A "$test_images_report")" ]]; then
+              echo "test_images_report=${test_images_report}" >> "$GITHUB_OUTPUT"
+            fi
           else
             echo "No $test_images_hook found, skipping tests."
           fi
@@ -369,11 +384,13 @@ jobs:
   retag-images:
     name: Retag images of flavor ${{ matrix.flavor.id || 'default' }}
     needs: [get-flavors, test-images]
-    if: ${{ inputs.skip_push != true }}
+    # Skip in case we would have an empty matrix. Note that in this case it also skips all downstream jobs...
+    # We need the always() && !cancelled() && !failure() because the test-images may have been skipped (which is fine). See https://github.com/actions/runner/issues/491#issuecomment-1507495166
+    if: ${{ always() && !cancelled() && !failure() && toJSON(fromJson(needs.get-flavors.outputs.result).flavorsToRetag) != '[]' }}
     strategy:
       fail-fast: false
       matrix:
-        flavor: ${{ fromJson(needs.get-flavors.outputs.result).flavors }}
+        flavor: ${{ fromJson(needs.get-flavors.outputs.result).flavorsToRetag }}
     # Do not run this on self-hosted, as it is faster and shouldn't be blocking anything
     # runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }}
     runs-on: "ubuntu-22.04"
@@ -410,7 +427,7 @@ jobs:
           echo "image_tag=$image_tag"
           echo "image_tag_branch_name=$image_tag_branch_name"
 
-          for repository_name in $(jq -r '.ecr_respositories[]' <<< "$FLAVOR"); do
+          for repository_name in $(jq -r '.ecr_repositories[]' <<< "$FLAVOR"); do
             IMAGE_META=$(aws ecr describe-images --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --arg var "${image_tag_branch_name}" '.imageDetails[0].imageTags | index( $var )')
             if [[ -z "${IMAGE_META}" || ${IMAGE_META} == "null" ]]; then
               MANIFEST=$(aws ecr batch-get-image --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --raw-output --join-output '.images[0].imageManifest')
@@ -428,9 +445,9 @@ jobs:
 
   push-to-repositories:
     name: Push images to push targets
-    # if? When should we do this? Always? Only for certain branches? If so, how should we define that, in the config.json?
-    if: ${{ inputs.skip_push != true && fromJson(needs.get-flavors.outputs.result).push_to != '' }}
     needs: [retag-images, get-flavors]
+    # if? When should we do this? Always? Only for certain branches? If so, how should we define that, in the config.json?
+    if: ${{ fromJson(needs.get-flavors.outputs.result).skip_push != true && fromJson(needs.get-flavors.outputs.result).push_to != '' }}
     uses: datavisyn/github-workflows/.github/workflows/build-docker-artifacts-trigger-push.yml@main
     secrets: inherit
     with: