test runner tags

Author: lucasssvaz

.github/scripts/tests_matrix.sh

@@ -12,11 +12,12 @@ if [[ $IS_PR != 'true' ]] || [[ $PERFORMANCE_ENABLED == 'true' ]]; then
     #qemu_types+=",'performance'"
 fi
 
-targets="'esp32','esp32s2','esp32s3','esp32c3','esp32c6','esp32h2','esp32p4'"
+targets="'esp32','esp32s2','esp32s3','esp32c3','esp32c5','esp32c6','esp32h2','esp32p4'"
 
 mkdir -p info
 
 echo "[$wokwi_types]" > info/wokwi_types.txt
+echo "[$hw_types]" > info/hw_types.txt
 echo "[$targets]" > info/targets.txt
 
 {

.github/workflows/tests.yml

@@ -89,9 +89,6 @@ jobs:
       type: ${{ matrix.type }}
       chip: ${{ matrix.chip }}
 
-  # Hardware tests now run via workflow_run trigger in tests_hw_wokwi.yml
-  # This provides secure access to GitLab secrets while maintaining the same functionality
-
   # This job is disabled for now
   call-qemu-tests:
     name: QEMU
@@ -107,4 +104,4 @@ jobs:
       type: ${{ matrix.type }}
       chip: ${{ matrix.chip }}
 
-  # Wokwi tests are run after this workflow as it needs access to secrets
+  # Hardware and Wokwi tests are run after this workflow as they need access to secrets

.github/workflows/tests_hw_wokwi.yml

@@ -10,6 +10,10 @@ on:
 permissions:
   contents: read
 
+env:
+  #TESTS_BRANCH: "master" # Branch that will be checked out to run the tests
+  TESTS_BRANCH: "ci/hw_gitlab"
+
 jobs:
   get-artifacts:
     name: Get required artifacts
@@ -22,7 +26,8 @@ jobs:
       ref: ${{ steps.set-ref.outputs.ref }}
       base: ${{ steps.set-ref.outputs.base }}
       targets: ${{ steps.set-ref.outputs.targets }}
-      types: ${{ steps.set-ref.outputs.types }}
+      wokwi_types: ${{ steps.set-ref.outputs.wokwi_types }}
+      hw_types: ${{ steps.set-ref.outputs.hw_types }}
       hw_tests_enabled: ${{ steps.set-ref.outputs.hw_tests_enabled }}
       push_time: ${{ steps.set-ref.outputs.push_time }}
     steps:
@@ -62,7 +67,7 @@ jobs:
           name: matrix_info
           path: artifacts/matrix_info
 
-      - name: Try to read PR number
+      - name: Get info
         id: set-ref
         run: |
           pr_num=$(jq -r '.pull_request.number' artifacts/event_file/event.json | tr -cd "[:digit:]")
@@ -102,12 +107,14 @@ jobs:
             push_time=""
           fi
 
-          types=$(cat artifacts/matrix_info/wokwi_types.txt | tr -cd "[:alpha:],[]'")
+          wokwi_types=$(cat artifacts/matrix_info/wokwi_types.txt | tr -cd "[:alpha:],[]'")
+          hw_types=$(cat artifacts/matrix_info/hw_types.txt | tr -cd "[:alpha:],[]'")
           targets=$(cat artifacts/matrix_info/targets.txt | tr -cd "[:alnum:],[]'")
 
           echo "base = $base"
           echo "targets = $targets"
-          echo "types = $types"
+          echo "wokwi_types = $wokwi_types"
+          echo "hw_types = $hw_types"
           echo "pr_num = $pr_num"
           echo "hw_tests_enabled = $hw_tests_enabled"
           echo "push_time = $push_time"
@@ -145,7 +152,8 @@ jobs:
           echo "pr_num=$pr_num" >> $GITHUB_OUTPUT
           echo "base=$base" >> $GITHUB_OUTPUT
           echo "targets=$targets" >> $GITHUB_OUTPUT
-          echo "types=$types" >> $GITHUB_OUTPUT
+          echo "wokwi_types=$wokwi_types" >> $GITHUB_OUTPUT
+          echo "hw_types=$hw_types" >> $GITHUB_OUTPUT
           echo "ref=$ref" >> $GITHUB_OUTPUT
           echo "hw_tests_enabled=$hw_tests_enabled" >> $GITHUB_OUTPUT
           echo "push_time=$push_time" >> $GITHUB_OUTPUT
@@ -189,7 +197,7 @@ jobs:
             core.info(`${name} is ${state}`);
 
   hardware-test:
-    name: Hardware ${{ matrix.chip }} ${{ matrix.type }} tests
+    name: Internal Hardware Tests
     if: |
       (github.event.workflow_run.conclusion == 'success' ||
       github.event.workflow_run.conclusion == 'failure' ||
@@ -198,15 +206,10 @@ jobs:
     runs-on: ubuntu-latest
     needs: get-artifacts
     env:
-      id: ${{ needs.get-artifacts.outputs.ref }}-${{ github.event.workflow_run.head_sha || github.sha }}-${{ matrix.chip }}-${{ matrix.type }}
+      id: ${{ needs.get-artifacts.outputs.ref }}-${{ github.event.workflow_run.head_sha || github.sha }}
     permissions:
       actions: read
       statuses: write
-    strategy:
-      fail-fast: false
-      matrix:
-        type: ${{ fromJson(needs.get-artifacts.outputs.types) }}
-        chip: ${{ fromJson(needs.get-artifacts.outputs.targets) }}
     steps:
       - name: Report pending
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
@@ -219,7 +222,7 @@ jobs:
             core.debug(`repo: ${repo}`);
             core.debug(`sha: ${sha}`);
             const { context: name, state } = (await github.rest.repos.createCommitStatus({
-              context: 'Runtime Tests / Hardware (${{ matrix.type }}, ${{ matrix.chip }}) / Hardware ${{ matrix.chip }} ${{ matrix.type }} tests (${{ github.event.workflow_run.event }} -> workflow_run)',
+              context: 'Runtime Tests / Internal Hardware Tests (${{ github.event.workflow_run.event }} -> workflow_run)',
               owner: owner,
               repo: repo,
               sha: sha,
@@ -258,8 +261,7 @@ jobs:
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         run: |
           # A webhook to sync the repository is sent to GitLab when a commit is pushed to GitHub
-          # In the worst case, it can take up to 5 minutes for the sync to complete
-          # We wait for 6 minutes after the push to GitHub to be safe
+          # We wait for 10 minutes after the push to GitHub to be safe
 
           echo "Ensuring GitLab sync has completed before triggering pipeline..."
 
@@ -278,8 +280,8 @@ jobs:
 
               echo "Elapsed time since push: ${elapsed_minutes} minutes"
 
-              if [ $elapsed_minutes -lt 6 ]; then
-                wait_time=$(( (6 - elapsed_minutes) * 60 ))
+              if [ $elapsed_minutes -lt 10 ]; then
+                wait_time=$(( (10 - elapsed_minutes) * 60 ))
                 echo "Waiting ${wait_time} seconds for GitLab sync to complete..."
                 sleep $wait_time
               else
@@ -302,30 +304,20 @@ jobs:
         with:
           ref: ${{ needs.get-artifacts.outputs.base || github.ref }}
 
-      - name: Get test binaries
-        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
-        uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          run-id: ${{ github.event.workflow_run.id }}
-          name: test-bin-${{ matrix.chip }}-${{ matrix.type }}
-          path: test-binaries
-
       - name: Trigger GitLab Pipeline and Download Artifacts
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         uses: digital-blueprint/[email protected]
         id: gitlab-trigger
         with:
           host: ${{ secrets.GITLAB_URL }}
           id: ${{ secrets.GITLAB_PROJECT_ID }}
-          ref: 'ci/hw_gitlab'
-          #ref: 'master'
+          ref: ${{ env.TESTS_BRANCH }}
           trigger_token: ${{ secrets.GITLAB_TRIGGER_TOKEN }}
           access_token: ${{ secrets.GITLAB_ACCESS_TOKEN }}
           download_artifacts: 'true'
           download_artifacts_on_failure: 'true'
           download_path: './gitlab-artifacts'
-          variables: '{"TEST_TYPE":"${{ matrix.type }}","TEST_CHIP":"${{ matrix.chip }}","PIPELINE_ID":"${{ env.id }}","BINARIES_RUN_ID":"${{ github.event.workflow_run.id }}","GITHUB_REPOSITORY":"${{ github.repository }}"}'
+          variables: '{"TEST_TYPES":"${{ needs.get-artifacts.outputs.hw_types }}","TEST_CHIPS":"${{ needs.get-artifacts.outputs.targets }}","PIPELINE_ID":"${{ env.id }}","BINARIES_RUN_ID":"${{ github.event.workflow_run.id }}","GITHUB_REPOSITORY":"${{ github.repository }}"}'
 
       - name: Process Downloaded Artifacts
         if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }}
@@ -356,7 +348,7 @@ jobs:
             echo "No artifacts were downloaded from GitLab"
           fi
 
-      - name: Upload ${{ matrix.chip }} ${{ matrix.type }} hardware results as cache
+      - name: Upload hardware results as cache
         uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         if: steps.check-tests.outputs.enabled == 'true' && needs.get-artifacts.outputs.pr_num
         with:
@@ -365,11 +357,11 @@ jobs:
             tests/**/*.xml
             tests/**/result_*.json
 
-      - name: Upload ${{ matrix.chip }} ${{ matrix.type }} hardware results as artifacts
+      - name: Upload hardware results as artifacts
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         if: always()
         with:
-          name: test-results-hw-${{ matrix.chip }}-${{ matrix.type }}
+          name: test-results-hw
           overwrite: true
           path: |
             tests/**/*.xml
@@ -387,7 +379,7 @@ jobs:
             core.debug(`repo: ${repo}`);
             core.debug(`sha: ${sha}`);
             const { context: name, state } = (await github.rest.repos.createCommitStatus({
-              context: 'Runtime Tests / Hardware (${{ matrix.type }}, ${{ matrix.chip }}) / Hardware ${{ matrix.chip }} ${{ matrix.type }} tests (${{ github.event.workflow_run.event }} -> workflow_run)',
+              context: 'Runtime Tests / Internal Hardware Tests (${{ github.event.workflow_run.event }} -> workflow_run)',
               owner: owner,
               repo: repo,
               sha: sha,
@@ -412,7 +404,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        type: ${{ fromJson(needs.get-artifacts.outputs.types) }}
+        type: ${{ fromJson(needs.get-artifacts.outputs.wokwi_types) }}
         chip: ${{ fromJson(needs.get-artifacts.outputs.targets) }}
     steps:
       - name: Report pending

.github/workflows/tests_results.yml

@@ -11,32 +11,29 @@ permissions:
   contents: read
 
 jobs:
-  unit-test-results:
-    name: Unit Test Results
-    if: |
-      github.event.workflow_run.conclusion == 'success' ||
-      github.event.workflow_run.conclusion == 'failure' ||
-      github.event.workflow_run.conclusion == 'timed_out'
+  get-artifacts:
+    name: Get artifacts
     runs-on: ubuntu-latest
-    permissions:
-      actions: write
-      statuses: write
-      checks: write
-      pull-requests: write
-      contents: write
+    outputs:
+      original_event: ${{ steps.get-info.outputs.original_event }}
+      original_action: ${{ steps.get-info.outputs.original_action }}
+      original_sha: ${{ steps.get-info.outputs.original_sha }}
+      original_ref: ${{ steps.get-info.outputs.original_ref }}
+      original_conclusion: ${{ steps.get-info.outputs.original_conclusion }}
+      original_run_id: ${{ steps.get-info.outputs.original_run_id }}
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          ref: gh-pages
-
       - name: Download and Extract Artifacts
         uses: dawidd6/action-download-artifact@07ab29fd4a977ae4d2b275087cf67563dfdf0295 # v9
         with:
           run_id: ${{ github.event.workflow_run.id }}
           path: ./artifacts
 
       - name: Get original info
+        id: get-info
         run: |
+          echo "Artifacts:"
+          ls -laR ./artifacts
+
           original_event=$(cat ./artifacts/parent-artifacts/event.txt)
           original_action=$(cat ./artifacts/parent-artifacts/action.txt)
           original_sha=$(cat ./artifacts/parent-artifacts/sha.txt)
@@ -64,12 +61,12 @@ jobs:
           # Run ID: Allow numeric characters
           original_run_id=$(echo "$original_run_id" | tr -cd '[:digit:]')
 
-          echo "original_event=$original_event" >> $GITHUB_ENV
-          echo "original_action=$original_action" >> $GITHUB_ENV
-          echo "original_sha=$original_sha" >> $GITHUB_ENV
-          echo "original_ref=$original_ref" >> $GITHUB_ENV
-          echo "original_conclusion=$original_conclusion" >> $GITHUB_ENV
-          echo "original_run_id=$original_run_id" >> $GITHUB_ENV
+          echo "original_event=$original_event" >> $GITHUB_OUTPUT
+          echo "original_action=$original_action" >> $GITHUB_OUTPUT
+          echo "original_sha=$original_sha" >> $GITHUB_OUTPUT
+          echo "original_ref=$original_ref" >> $GITHUB_OUTPUT
+          echo "original_conclusion=$original_conclusion" >> $GITHUB_OUTPUT
+          echo "original_run_id=$original_run_id" >> $GITHUB_OUTPUT
 
           echo "original_event = $original_event"
           echo "original_action = $original_action"
@@ -80,17 +77,37 @@ jobs:
 
       - name: Print links to other runs
         run: |
-          echo "Build and QEMU tests: https://github.com/${{ github.repository }}/actions/runs/${{ env.original_run_id }}"
+          echo "Build and QEMU tests: https://github.com/${{ github.repository }}/actions/runs/${{ steps.get-info.outputs.original_run_id }}"
           echo "Hardware and Wokwi tests: https://github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}"
 
+  unit-test-results:
+    name: Unit Test Results
+    needs: get-artifacts
+    if: |
+      github.event.workflow_run.conclusion == 'success' ||
+      github.event.workflow_run.conclusion == 'failure' ||
+      github.event.workflow_run.conclusion == 'timed_out'
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      statuses: write
+      checks: write
+      pull-requests: write
+      contents: write
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: gh-pages
+
       - name: Publish Unit Test Results
         uses: EnricoMi/publish-unit-test-result-action@170bf24d20d201b842d7a52403b73ed297e6645b # v2.18.0
         with:
-          commit: ${{ env.original_sha }}
+          commit: ${{ needs.get-artifacts.outputs.original_sha }}
           event_file: ./artifacts/parent-artifacts/event_file/event.json
-          event_name: ${{ env.original_event }}
+          event_name: ${{ needs.get-artifacts.outputs.original_event }}
           files: ./artifacts/**/*.xml
           action_fail: true
+          action_fail_on_inconclusive: true
           compare_to_earlier_commit: false
           json_file: ./unity_results.json
           json_suite_details: true
@@ -105,7 +122,13 @@ jobs:
             ./unity_results.json
 
       - name: Fail if tests failed
-        if: ${{ env.original_conclusion == 'failure' || env.original_conclusion == 'timed_out' || github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out' }}
+        if: |
+          needs.get-artifacts.outputs.original_conclusion == 'failure' ||
+          needs.get-artifacts.outputs.original_conclusion == 'cancelled' ||
+          needs.get-artifacts.outputs.original_conclusion == 'timed_out' ||
+          github.event.workflow_run.conclusion == 'failure' ||
+          github.event.workflow_run.conclusion == 'cancelled' ||
+          github.event.workflow_run.conclusion == 'timed_out'
         run: exit 1
 
       - name: Clean up caches
@@ -162,12 +185,24 @@ jobs:
             core.info(`${name} is ${state}`);
 
       - name: Generate report
-        if: ${{ !cancelled() && (env.original_event == 'schedule' || env.original_event == 'workflow_dispatch') }} # codespell:ignore cancelled
+        if: |
+          (!cancelled() &&
+           needs.get-artifacts.outputs.original_conclusion != 'cancelled' &&
+           github.event.workflow_run.conclusion != 'cancelled') &&
+          (needs.get-artifacts.outputs.original_event == 'schedule' ||
+           needs.get-artifacts.outputs.original_event == 'workflow_dispatch')
         env:
           REPORT_FILE: ./runtime-test-results/RUNTIME_TEST_RESULTS.md
           WOKWI_RUN_ID: ${{ github.event.workflow_run.id }}
-          BUILD_RUN_ID: ${{ env.original_run_id }}
-          IS_FAILING: ${{ env.original_conclusion == 'failure' || env.original_conclusion == 'timed_out' || github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out' || job.status == 'failure' }}
+          BUILD_RUN_ID: ${{ needs.get-artifacts.outputs.original_run_id }}
+          IS_FAILING: |
+            needs.get-artifacts.outputs.original_conclusion == 'failure' ||
+            needs.get-artifacts.outputs.original_conclusion == 'cancelled' ||
+            needs.get-artifacts.outputs.original_conclusion == 'timed_out' ||
+            github.event.workflow_run.conclusion == 'failure' ||
+            github.event.workflow_run.conclusion == 'cancelled' ||
+            github.event.workflow_run.conclusion == 'timed_out' ||
+            job.status == 'failure'
         run: |
           rm -rf artifacts $REPORT_FILE
           mv -f ./unity_results.json ./runtime-test-results/unity_results.json
@@ -176,7 +211,12 @@ jobs:
           mv -f ./test_results.json ./runtime-test-results/test_results.json
 
       - name: Generate badge
-        if: ${{ !cancelled() && (env.original_event == 'schedule' || env.original_event == 'workflow_dispatch') }} # codespell:ignore cancelled
+        if: |
+          (!cancelled() &&
+           needs.get-artifacts.outputs.original_conclusion != 'cancelled' &&
+           github.event.workflow_run.conclusion != 'cancelled') &&
+          (needs.get-artifacts.outputs.original_event == 'schedule' ||
+           needs.get-artifacts.outputs.original_event == 'workflow_dispatch')
         uses: jaywcjlove/generated-badges@0e078ae4d4bab3777ea4f137de496ab44688f5ad # v1.0.13
         with:
           label: Runtime Tests
@@ -186,7 +226,12 @@ jobs:
           style: flat
 
       - name: Push badge
-        if: ${{ !cancelled() && (env.original_event == 'schedule' || env.original_event == 'workflow_dispatch') }} # codespell:ignore cancelled
+        if: |
+          (!cancelled() &&
+           needs.get-artifacts.outputs.original_conclusion != 'cancelled' &&
+           github.event.workflow_run.conclusion != 'cancelled') &&
+          (needs.get-artifacts.outputs.original_event == 'schedule' ||
+           needs.get-artifacts.outputs.original_event == 'workflow_dispatch')
         run: |
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"