From eeef05c02fa97e5477e9d7584745a1e78e50461e Mon Sep 17 00:00:00 2001 From: KornevNikita Date: Mon, 23 Dec 2024 06:07:39 -0800 Subject: [PATCH 1/4] [CI] Build & run CTS separately Gen12 takes ~90min to build and run SYCL-CTS, where run takes less than 10mins. This patch updates workflows to build the suite on PVC and then upload artifacts to required machines. --- .github/workflows/sycl-linux-run-tests.yml | 42 ++++++++++++-- .github/workflows/sycl-nightly.yml | 65 +++++++++++++++++----- 2 files changed, 89 insertions(+), 18 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 4eff7a47c6c15..dc74fb0416e21 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -79,7 +79,7 @@ on: type: string default: 'default' retention-days: - description: 'E2E binaries artifact retention period.' + description: 'E2E/SYCL-CTS binaries artifact retention period.' type: string default: 1 @@ -102,6 +102,16 @@ on: default: 'false' required: False + cts_build_only: + type: string + default: 'false' + required: False + + sycl_cts_artifact: + type: string + default: '' + required: False + workflow_dispatch: inputs: runner: @@ -339,7 +349,7 @@ jobs: fi exit $exit_code - name: Build SYCL CTS tests - if: inputs.tests_selector == 'cts' + if: inputs.tests_selector == 'cts' && inputs.sycl_cts_artifact == '' env: CMAKE_EXTRA_ARGS: ${{ inputs.extra_cmake_args }} run: | @@ -371,9 +381,33 @@ jobs: # "test_conformance" target skips building "test_all" executable. ninja -C build-cts -k0 $( [ -n "$CTS_TESTS_TO_BUILD" ] && echo "$CTS_TESTS_TO_BUILD" || echo "test_conformance") + - name: Pack SYCL-CTS binaries + if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.cts_build_only == 'true' + run: tar -I 'zstd -9' -cf sycl_cts_bin.tar.zst -C ./build-cts/bin . + + - name: Upload SYCL-CTS binaries + if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.cts_build_only == 'true' + uses: actions/upload-artifact@v4 + with: + name: sycl_cts_bin + path: sycl_cts_bin.tar.zst + retention-days: ${{ inputs.retention-days }} + + - name: Download SYCL-CTS binaries + if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.sycl_cts_artifact != '' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.sycl_cts_artifact }} + + - name: Extract SYCL-CTS binaries + if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.sycl_cts_artifact != '' + run: | + mkdir -p build-cts/bin + tar -I 'zstd' -xf sycl_cts_bin.tar.zst -C build-cts/bin + - name: SYCL CTS List devices # Proceed with execution even if the 'build' step did not succeed. - if: inputs.tests_selector == 'cts' && (success() || failure()) + if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.cts_build_only != 'true' env: ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} run: | @@ -381,7 +415,7 @@ jobs: - name: Run SYCL CTS tests # Proceed with execution even if the previous two steps did not succeed. - if: inputs.tests_selector == 'cts' && (success() || failure()) + if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.cts_build_only != 'true' env: ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} # This job takes ~100min usually. But sometimes some test isn't diff --git a/.github/workflows/sycl-nightly.yml b/.github/workflows/sycl-nightly.yml index 1c1c5c6d1f398..3fd9d2a463e3d 100644 --- a/.github/workflows/sycl-nightly.yml +++ b/.github/workflows/sycl-nightly.yml @@ -88,20 +88,6 @@ jobs: image_options: -u 1001 target_devices: opencl:cpu tests_selector: e2e - - - name: SYCL-CTS on OCL CPU - runner: '["Linux", "gen12"]' - image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest - image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN - target_devices: opencl:cpu - tests_selector: cts - - - name: SYCL-CTS on L0 gen12 - runner: '["Linux", "gen12"]' - image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest - image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN - target_devices: level_zero:gpu - tests_selector: cts uses: ./.github/workflows/sycl-linux-run-tests.yml with: name: ${{ matrix.name }} @@ -174,6 +160,57 @@ jobs: with: mode: stop + build-sycl-cts: + needs: ubuntu2204_build + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: Build SYCL-CTS + runner: '["Linux", "build"]' + cts_build_only: 'true' + image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + tests_selector: cts + ref: ${{ github.sha }} + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + + run-sycl-cts: + needs: [ubuntu2204_build, build-sycl-cts] + if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }} + strategy: + fail-fast: false + matrix: + include: + - name: SYCL-CTS on OCL CPU + runner: '["Linux", "gen12"]' + image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + target_devices: opencl:cpu + tests_selector: cts + + - name: SYCL-CTS on L0 gen12 + runner: '["Linux", "gen12"]' + image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest + image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN + target_devices: level_zero:gpu + tests_selector: cts + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: ${{ matrix.name }} + runner: ${{ matrix.runner }} + image: ${{ matrix.image }} + image_options: ${{ matrix.image_options }} + target_devices: ${{ matrix.target_devices }} + tests_selector: ${{ matrix.tests_selector }} + ref: ${{ github.sha }} + merge_ref: '' + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }} + sycl_cts_artifact: sycl_cts_bin + nightly_build_upload: name: Nightly Build Upload if: ${{ github.ref_name == 'sycl' }} From 24cd62ffcc2e451a6354be236bfa1eea4f4c5592 Mon Sep 17 00:00:00 2001 From: KornevNikita Date: Mon, 30 Dec 2024 04:57:07 -0800 Subject: [PATCH 2/4] Filter test categories --- .github/workflows/sycl-linux-run-tests.yml | 21 ++++++++++++++++++++- devops/cts_exclude_filter_L0_GPU | 2 ++ devops/cts_exclude_filter_OCL_CPU | 2 ++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index dc74fb0416e21..09428e007a6d6 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -156,7 +156,7 @@ on: LIT_OPTS won't work as we redefine it as part of this workflow. For SYCL CTS - CTS_TESTS_TO_BUILD to specify which categories to - build. + build, e.g. {"CTS_TESTS_TO_BUILD":"test_category1 test_category2..."}. Format: '{"VAR1":"VAL1","VAR2":"VAL2",...}' default: '{}' @@ -413,6 +413,25 @@ jobs: run: | ./build-cts/bin/* --list-devices + # If the suite was built on another machine then it also contains excluded + # categories. Check the filter file and remove them. + - name: Filter SYCL CTS test categories + if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.sycl_cts_artifact != '' + shell: bash + run: | + cts_exclude_filter="" + if [ "${{ contains(inputs.target_devices, 'opencl:cpu') }}" = "true" ]; then + cts_exclude_filter=$PWD/devops/cts_exclude_filter_OCL_CPU + elif [ "${{ contains(inputs.target_devices, 'level_zero:gpu') }}" = "true" ]; then + cts_exclude_filter=$PWD/devops/cts_exclude_filter_L0_GPU + fi + + while IFS= read -r line; do + if [[ $line != \#* ]]; then + rm "./build-cts/bin/test_$line" + fi + done < "$cts_exclude_filter" + - name: Run SYCL CTS tests # Proceed with execution even if the previous two steps did not succeed. if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.cts_build_only != 'true' diff --git a/devops/cts_exclude_filter_L0_GPU b/devops/cts_exclude_filter_L0_GPU index d3172efdae4ab..fa45b9ac1e5cf 100644 --- a/devops/cts_exclude_filter_L0_GPU +++ b/devops/cts_exclude_filter_L0_GPU @@ -1,2 +1,4 @@ +# Please use "#" to add comments here. +# Do not delete the file even if it's empty. # CMPLRTST-26179 device diff --git a/devops/cts_exclude_filter_OCL_CPU b/devops/cts_exclude_filter_OCL_CPU index e69de29bb2d1d..44d3870b88048 100644 --- a/devops/cts_exclude_filter_OCL_CPU +++ b/devops/cts_exclude_filter_OCL_CPU @@ -0,0 +1,2 @@ +# Please use "#" to add comments here. +# Do not delete the file even if it's empty. From 06bd20b070fd551188724aa67dbfa1b1f86f04c8 Mon Sep 17 00:00:00 2001 From: KornevNikita Date: Thu, 2 Jan 2025 03:40:10 -0800 Subject: [PATCH 3/4] update conditions --- .github/workflows/sycl-linux-run-tests.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 09428e007a6d6..24b4fb6119c27 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -382,11 +382,11 @@ jobs: ninja -C build-cts -k0 $( [ -n "$CTS_TESTS_TO_BUILD" ] && echo "$CTS_TESTS_TO_BUILD" || echo "test_conformance") - name: Pack SYCL-CTS binaries - if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.cts_build_only == 'true' + if: always() && !cancelled() && inputs.cts_build_only == 'true' run: tar -I 'zstd -9' -cf sycl_cts_bin.tar.zst -C ./build-cts/bin . - name: Upload SYCL-CTS binaries - if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.cts_build_only == 'true' + if: always() && !cancelled() && inputs.cts_build_only == 'true' uses: actions/upload-artifact@v4 with: name: sycl_cts_bin @@ -394,20 +394,20 @@ jobs: retention-days: ${{ inputs.retention-days }} - name: Download SYCL-CTS binaries - if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.sycl_cts_artifact != '' + if: inputs.sycl_cts_artifact != '' uses: actions/download-artifact@v4 with: name: ${{ inputs.sycl_cts_artifact }} - name: Extract SYCL-CTS binaries - if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.sycl_cts_artifact != '' + if: inputs.sycl_cts_artifact != '' run: | mkdir -p build-cts/bin tar -I 'zstd' -xf sycl_cts_bin.tar.zst -C build-cts/bin - name: SYCL CTS List devices # Proceed with execution even if the 'build' step did not succeed. - if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.cts_build_only != 'true' + if: inputs.tests_selector == 'cts' && (always() && !cancelled()) && inputs.cts_build_only != 'true' env: ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} run: | @@ -416,7 +416,7 @@ jobs: # If the suite was built on another machine then it also contains excluded # categories. Check the filter file and remove them. - name: Filter SYCL CTS test categories - if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.sycl_cts_artifact != '' + if: inputs.sycl_cts_artifact != '' shell: bash run: | cts_exclude_filter="" @@ -434,7 +434,7 @@ jobs: - name: Run SYCL CTS tests # Proceed with execution even if the previous two steps did not succeed. - if: inputs.tests_selector == 'cts' && (success() || failure()) && inputs.cts_build_only != 'true' + if: inputs.tests_selector == 'cts' && (always() && !cancelled()) && inputs.cts_build_only != 'true' env: ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} # This job takes ~100min usually. But sometimes some test isn't From a67ac8d0d4155ae7cea9eae982193025bcc4bac9 Mon Sep 17 00:00:00 2001 From: KornevNikita Date: Thu, 2 Jan 2025 08:51:02 -0800 Subject: [PATCH 4/4] apply suggestions --- .github/workflows/sycl-linux-run-tests.yml | 25 ++++++++++++++-------- .github/workflows/sycl-nightly.yml | 3 ++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 24b4fb6119c27..49e58e044a0f5 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -102,10 +102,13 @@ on: default: 'false' required: False - cts_build_only: + cts_testing_mode: + description: | + Testing mode to run SYCL-CTS in, can be either `full`, `build-only` + or `run-only`. In `build-only` mode an artifact of the CTS binaries + will be uploaded. type: string - default: 'false' - required: False + default: 'full' sycl_cts_artifact: type: string @@ -382,11 +385,11 @@ jobs: ninja -C build-cts -k0 $( [ -n "$CTS_TESTS_TO_BUILD" ] && echo "$CTS_TESTS_TO_BUILD" || echo "test_conformance") - name: Pack SYCL-CTS binaries - if: always() && !cancelled() && inputs.cts_build_only == 'true' + if: always() && !cancelled() && inputs.cts_testing_mode == 'build-only' run: tar -I 'zstd -9' -cf sycl_cts_bin.tar.zst -C ./build-cts/bin . - name: Upload SYCL-CTS binaries - if: always() && !cancelled() && inputs.cts_build_only == 'true' + if: always() && !cancelled() && inputs.cts_testing_mode == 'build-only' uses: actions/upload-artifact@v4 with: name: sycl_cts_bin @@ -407,14 +410,18 @@ jobs: - name: SYCL CTS List devices # Proceed with execution even if the 'build' step did not succeed. - if: inputs.tests_selector == 'cts' && (always() && !cancelled()) && inputs.cts_build_only != 'true' + if: inputs.tests_selector == 'cts' && (always() && !cancelled()) && inputs.cts_testing_mode != 'build-only' env: ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} run: | ./build-cts/bin/* --list-devices - # If the suite was built on another machine then it also contains excluded - # categories. Check the filter file and remove them. + # If the suite was built on another machine then the build contains the full + # set of tests. We have special files to filter out some test categories, + # see "devops/cts_exclude_filter_*". Each configuration has its own file, e.g. + # there is "cts_exclude_filter_OCL_CPU" for opencl:cpu device. Therefore, + # these files may differ from each other, so when there is a pre-built set of + # tests, we need to filter it according to the filter-file. - name: Filter SYCL CTS test categories if: inputs.sycl_cts_artifact != '' shell: bash @@ -434,7 +441,7 @@ jobs: - name: Run SYCL CTS tests # Proceed with execution even if the previous two steps did not succeed. - if: inputs.tests_selector == 'cts' && (always() && !cancelled()) && inputs.cts_build_only != 'true' + if: inputs.tests_selector == 'cts' && (always() && !cancelled()) && inputs.cts_testing_mode != 'build-only' env: ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }} # This job takes ~100min usually. But sometimes some test isn't diff --git a/.github/workflows/sycl-nightly.yml b/.github/workflows/sycl-nightly.yml index 3fd9d2a463e3d..e47c02591dccc 100644 --- a/.github/workflows/sycl-nightly.yml +++ b/.github/workflows/sycl-nightly.yml @@ -167,7 +167,7 @@ jobs: with: name: Build SYCL-CTS runner: '["Linux", "build"]' - cts_build_only: 'true' + cts_testing_mode: 'build-only' image: ghcr.io/intel/llvm/ubuntu2404_intel_drivers:latest image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN tests_selector: cts @@ -200,6 +200,7 @@ jobs: with: name: ${{ matrix.name }} runner: ${{ matrix.runner }} + cts_testing_mode: 'run-only' image: ${{ matrix.image }} image_options: ${{ matrix.image_options }} target_devices: ${{ matrix.target_devices }}