From 0141cbfee4494e479c6b3b768a9c73727429fed7 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 05:03:48 +0000 Subject: [PATCH 01/65] switch to cibuildwheel + some cleanups --- .github/actions/build/action.yml | 48 +++++++------------------ .github/workflows/gh-build-and-test.yml | 15 ++++---- .github/workflows/gh-build.yml | 7 +--- 3 files changed, 20 insertions(+), 50 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 952fb9cd1..e6d85bb7f 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -13,12 +13,6 @@ inputs: host-platform: required: true type: string - use-container: - required: true - type: boolean - docker-image: - type: string - required: true upload-enabled: required: true type: boolean @@ -29,36 +23,18 @@ inputs: runs: using: composite steps: - - - if: ${{ inputs.use-container }} - name: Build (in container) - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - - docker run \ - -e AWS_REGION \ - -e AWS_SESSION_TOKEN \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - -e GITHUB_TOKEN \ - -e BINDINGS_ARTIFACTS_DIR="$BINDINGS_ARTIFACTS_DIR" \ - -e CORE_ARTIFACTS_DIR="$CORE_ARTIFACTS_DIR" \ - -e UPLOAD_ENABLED="$UPLOAD_ENABLED" \ - -e USE_CUDA="$USE_CUDA" \ - -e REPO_DIR="$REPO_DIR" \ - -e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \ - -e PYTHON_VERSION="$PYTHON_VERSION" \ - -v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \ - -v "${{ env.BINDINGS_ARTIFACTS_DIR }}:${{ env.BINDINGS_ARTIFACTS_DIR }}" \ - -v "${{ env.CORE_ARTIFACTS_DIR }}:${{ env.CORE_ARTIFACTS_DIR }}" \ - --rm "${{ inputs.docker-image }}" \ - /bin/bash -c "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}" - - - if: ${{ !inputs.use-container }} - name: Build (without container) - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}" + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Build cuda.core wheel + uses: pypa/cibuildwheel@v2.22.0 + env: + CIBW_BUILD: cp312-* # FIXME: Only build on CPython 3.12 + CIBW_ARCHS_LINUX: "auto" + CIBW_REPAIR_WHEEL_COMMAND: "" # no repair + with: + package-dir: ./cuda_core/ + output-dir: ${{ env.BINDINGS_ARTIFACTS_DIR }} - name: Display structure of the bindings artifacts folder (post build) shell: bash --noprofile --norc -xeuo pipefail {0} diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml index ffc6f9594..eae1c25a1 100644 --- a/.github/workflows/gh-build-and-test.yml +++ b/.github/workflows/gh-build-and-test.yml @@ -1,16 +1,16 @@ on: workflow_call: inputs: - host-platform: + target-device: type: string required: true - target-device: + build-type: type: string required: true - build-mode: + host-platform: type: string required: true - build-type: + build-mode: type: string required: true upload-enabled: @@ -28,13 +28,12 @@ jobs: client-repo: ${{ github.event.repository.name }} target-device: ${{ inputs.target-device }} runs-on: ${{ (inputs.host-platform == 'linux-x64' && 'linux-amd64-cpu8') || - (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') }} + (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') || + (inputs.host-platform == 'win-x64' && 'linux-amd64-cpu8') }} build-type: ${{ inputs.build-type }} - use-container: ${{ inputs.host-platform == 'linux-x64' || - inputs.host-platform == 'linux-aarch64'}} host-platform: ${{ inputs.host-platform }} - dependencies-file: "" build-mode: ${{ inputs.build-mode }} upload-enabled: ${{ inputs.upload-enabled }} python-version: ${{ inputs.python-version }} + dependencies-file: "" secrets: inherit diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index c60e0c2ac..3a0d2dea3 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -16,9 +16,6 @@ on: required: true type: string description: One of ci / release - use-container: - required: true - type: boolean host-platform: required: true type: string @@ -38,7 +35,7 @@ on: jobs: build: - name: Build (${{ inputs.host-platform }}, ${{ inputs.target-device }}, ${{ inputs.build-type }}, CMake build-mode=${{ inputs.build-mode }}, Python "${{ inputs.python-version }}", Use container=${{ inputs.use-container }} ) + name: Build (${{ inputs.host-platform }}, ${{ inputs.target-device }}, ${{ inputs.build-type }}, build-mode=${{ inputs.build-mode }}, Python "${{ inputs.python-version }}") permissions: id-token: write # This is required for configure-aws-credentials @@ -69,6 +66,4 @@ jobs: build-type: ${{ inputs.build-type }} target-device: "${{ inputs.target-device }}" host-platform: ${{ inputs.host-platform }} - use-container: ${{ inputs.use-container }} - docker-image: "condaforge/miniforge3:latest" upload-enabled: ${{ inputs.upload-enabled }} From 79fd67afcbdc392532efeba67668363b42cd306e Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 05:23:24 +0000 Subject: [PATCH 02/65] try setting up Python manually --- .github/actions/build/action.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index e6d85bb7f..b2c870268 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -26,10 +26,17 @@ runs: - name: Check out the repo uses: actions/checkout@v4 + - name: Set up Python + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - name: Build cuda.core wheel uses: pypa/cibuildwheel@v2.22.0 env: CIBW_BUILD: cp312-* # FIXME: Only build on CPython 3.12 + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6" CIBW_ARCHS_LINUX: "auto" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair with: From 92283dc47791b9b622a7fa2f480f26e1a6133588 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 05:29:06 +0000 Subject: [PATCH 03/65] comment out un-needed code + propagate python-version --- .github/workflows/gh-build.yml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 3a0d2dea3..90d7a4d62 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -44,21 +44,21 @@ jobs: runs-on: ${{ inputs.runs-on }} steps: - - name: Checkout ${{ inputs.client-repo }} - uses: actions/checkout@v4 - with: - fetch-depth: 0 + #- name: Checkout ${{ inputs.client-repo }} + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 - - name: Setup - uses: ./.github/actions/setup - with: - client-repo: ${{ inputs.client-repo }} - build-type: ${{ inputs.build-type }} - target-device: "${{ inputs.target-device }}" - host-platform: ${{ inputs.host-platform }} - build-mode: ${{ inputs.build-mode }} - upload-enabled: ${{ inputs.upload-enabled }} - python-version: ${{ inputs.python-version }} + #- name: Setup + # uses: ./.github/actions/setup + # with: + # client-repo: ${{ inputs.client-repo }} + # build-type: ${{ inputs.build-type }} + # target-device: "${{ inputs.target-device }}" + # host-platform: ${{ inputs.host-platform }} + # build-mode: ${{ inputs.build-mode }} + # upload-enabled: ${{ inputs.upload-enabled }} + # python-version: ${{ inputs.python-version }} - name: Call build action uses: ./.github/actions/build @@ -67,3 +67,4 @@ jobs: target-device: "${{ inputs.target-device }}" host-platform: ${{ inputs.host-platform }} upload-enabled: ${{ inputs.upload-enabled }} + python-version: ${{ inputs.python-version }} From 6666e905714f16d6d1d7973071cadc51b2abb2aa Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 05:31:26 +0000 Subject: [PATCH 04/65] fix: need to check out first --- .github/actions/build/action.yml | 5 +---- .github/workflows/gh-build.yml | 8 ++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index b2c870268..2f5d5f6c5 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -23,10 +23,7 @@ inputs: runs: using: composite steps: - - name: Check out the repo - uses: actions/checkout@v4 - - - name: Set up Python + - name: Set up Python ${{ inputs.python-version }} id: setup-python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 90d7a4d62..5a26691ef 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -44,10 +44,10 @@ jobs: runs-on: ${{ inputs.runs-on }} steps: - #- name: Checkout ${{ inputs.client-repo }} - # uses: actions/checkout@v4 - # with: - # fetch-depth: 0 + - name: Checkout ${{ inputs.client-repo }} + uses: actions/checkout@v4 + with: + fetch-depth: 0 #- name: Setup # uses: ./.github/actions/setup From c96fef3b901c1ba64db23f2292ff4cb19c10baf8 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 05:42:08 +0000 Subject: [PATCH 05/65] only build natively; add -v; ensure targeting manylinux --- .github/actions/build/action.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 2f5d5f6c5..52ce69a03 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -32,10 +32,14 @@ runs: - name: Build cuda.core wheel uses: pypa/cibuildwheel@v2.22.0 env: - CIBW_BUILD: cp312-* # FIXME: Only build on CPython 3.12 + CIBW_BUILD: ${{ (inputs.python-version == '3.12' && "cp312-manylinux*") || + (inputs.python-version == '3.11' && "cp311-manylinux*") || + (inputs.python-version == '3.10' && "cp310-manylinux*") || + (inputs.python-version == '3.9' && "cp39-manylinux*") }} CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6" - CIBW_ARCHS_LINUX: "auto" + CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair + CIBW_BUILD_VERBOSITY: 1 with: package-dir: ./cuda_core/ output-dir: ${{ env.BINDINGS_ARTIFACTS_DIR }} From ea775722d50cc1a47d57a01a4f6c535f0ea92663 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 05:44:34 +0000 Subject: [PATCH 06/65] single quotes --- .github/actions/build/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 52ce69a03..7a7c6cc01 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -32,10 +32,10 @@ runs: - name: Build cuda.core wheel uses: pypa/cibuildwheel@v2.22.0 env: - CIBW_BUILD: ${{ (inputs.python-version == '3.12' && "cp312-manylinux*") || - (inputs.python-version == '3.11' && "cp311-manylinux*") || - (inputs.python-version == '3.10' && "cp310-manylinux*") || - (inputs.python-version == '3.9' && "cp39-manylinux*") }} + CIBW_BUILD: ${{ (inputs.python-version == '3.12' && 'cp312-manylinux*') || + (inputs.python-version == '3.11' && 'cp311-manylinux*') || + (inputs.python-version == '3.10' && 'cp310-manylinux*') || + (inputs.python-version == '3.9' && 'cp39-manylinux*') }} CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6" CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair From 508a12606a4a13b04511fda6387159ae3fb77840 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 05:52:59 +0000 Subject: [PATCH 07/65] restore env setup to get artifact dir --- .github/actions/setup/action.yml | 5 ----- .github/workflows/gh-build.yml | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index c2a8407c3..34111d99d 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -57,15 +57,10 @@ runs: PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" fi - PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.') - - echo "BINDINGS_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_bindings-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV echo "BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV - echo "CORE_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV echo "CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV - echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 5a26691ef..70f7772d6 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -49,16 +49,16 @@ jobs: with: fetch-depth: 0 - #- name: Setup - # uses: ./.github/actions/setup - # with: - # client-repo: ${{ inputs.client-repo }} - # build-type: ${{ inputs.build-type }} - # target-device: "${{ inputs.target-device }}" - # host-platform: ${{ inputs.host-platform }} - # build-mode: ${{ inputs.build-mode }} - # upload-enabled: ${{ inputs.upload-enabled }} - # python-version: ${{ inputs.python-version }} + - name: Setup environment variables + uses: ./.github/actions/setup + with: + client-repo: ${{ inputs.client-repo }} + build-type: ${{ inputs.build-type }} + target-device: "${{ inputs.target-device }}" + host-platform: ${{ inputs.host-platform }} + build-mode: ${{ inputs.build-mode }} + upload-enabled: ${{ inputs.upload-enabled }} + python-version: ${{ inputs.python-version }} - name: Call build action uses: ./.github/actions/build From beff50d94d44abbb2e0a724384fe788b6a909e2f Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 06:00:52 +0000 Subject: [PATCH 08/65] fix artifact dir --- .github/actions/build/action.yml | 34 ++++++++++++++++---------------- .github/actions/setup/action.yml | 4 ++-- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 7a7c6cc01..8081548a5 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -42,28 +42,28 @@ runs: CIBW_BUILD_VERBOSITY: 1 with: package-dir: ./cuda_core/ - output-dir: ${{ env.BINDINGS_ARTIFACTS_DIR }} + output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - - name: Display structure of the bindings artifacts folder (post build) - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - sudo chown -R $(whoami) ${{ env.BINDINGS_ARTIFACTS_DIR }} - ls -lahR ${{ env.BINDINGS_ARTIFACTS_DIR }} + #- name: Display structure of the bindings artifacts folder (post build) + # shell: bash --noprofile --norc -xeuo pipefail {0} + # run: | + # sudo chown -R $(whoami) ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + # ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - - name: Upload bindings build artifacts - uses: actions/upload-artifact@v4 - with: - name: ${{ env.BINDINGS_ARTIFACT_NAME }} - path: ${{ env.BINDINGS_ARTIFACTS_DIR }} + #- name: Upload bindings build artifacts + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} + # path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - - name: Display structure of the core artifacts folder (post build) + - name: List the cuda.core artifacts directory shell: bash --noprofile --norc -xeuo pipefail {0} run: | - sudo chown -R $(whoami) ${{ env.CORE_ARTIFACTS_DIR }} - ls -lahR ${{ env.CORE_ARTIFACTS_DIR }} + sudo chown -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - - name: Upload core build artifacts + - name: Upload cuda.core build artifacts uses: actions/upload-artifact@v4 with: - name: ${{ env.CORE_ARTIFACT_NAME }} - path: ${{ env.CORE_ARTIFACTS_DIR }} + name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} + path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 34111d99d..c4ab05b33 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -57,8 +57,8 @@ runs: PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" fi - echo "BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV - echo "CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV + echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV + echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV From a935da2ea266608107630a56ff046b52fb5cc88d Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 06:04:37 +0000 Subject: [PATCH 09/65] fix artifact name --- .github/actions/build/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 8081548a5..960b84193 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -65,5 +65,5 @@ runs: - name: Upload cuda.core build artifacts uses: actions/upload-artifact@v4 with: - name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} + name: cuda_core*.whl path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} From df982e5e5bc821d2355e3906b7db395c8bad76a4 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 06:24:03 +0000 Subject: [PATCH 10/65] restore & fix artifact name again --- .github/actions/build/action.yml | 6 ++++-- .github/actions/setup/action.yml | 10 +--------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 960b84193..bf121c1b6 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -65,5 +65,7 @@ runs: - name: Upload cuda.core build artifacts uses: actions/upload-artifact@v4 with: - name: cuda_core*.whl - path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + name: ${{ env.CUDA_CORE_ARTIFACT_NAME }} + path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl + if-no-files-found: error + overwrite: 'true' diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index c4ab05b33..fa24536d8 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -35,21 +35,12 @@ runs: - name: Set environment variables shell: bash --noprofile --norc -xeuo pipefail {0} run: | - - WITH_TESTS_STR='' - if [[ ("${{ inputs.upload-enabled }}" == "false") && ("${{ inputs.build-type }}" != "ci") ]]; then - WITH_TESTS_STR='-with_tests' - fi - TARGET_PLATFORM='linux-64' if [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then TARGET_PLATFORM='linux-aarch64' fi BUILD_MODE="${{ inputs.build-mode }}" - BUILD_MODE_STR="" - [ -n "${BUILD_MODE}" ] && BUILD_MODE_STR="-${BUILD_MODE}" - if [[ ("${BUILD_MODE}" == "") || ("${BUILD_MODE}" == "release") ]]; then # We upload release versions in the default folder. PKG_DIR="${TARGET_PLATFORM}" @@ -58,6 +49,7 @@ runs: fi echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV + echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${{ inputs.python-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV From 426cd583ae489a307de497a964521aa83cb6eb1a Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 06:34:15 +0000 Subject: [PATCH 11/65] build on all platforms! --- .github/actions/setup/action.yml | 10 ++++++++-- .github/workflows/ci-gh.yml | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index fa24536d8..a48d2f51e 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -35,9 +35,15 @@ runs: - name: Set environment variables shell: bash --noprofile --norc -xeuo pipefail {0} run: | - TARGET_PLATFORM='linux-64' - if [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then + # TODO: just align host-platform names with TARGET_PLATFORM... + if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then + TARGET_PLATFORM='linux-64' + elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then TARGET_PLATFORM='linux-aarch64' + elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then + TARGET_PLATFORM='win-64' + else + exit 1 fi BUILD_MODE="${{ inputs.build-mode }}" diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index d38cb8e30..cf1683257 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -18,6 +18,8 @@ jobs: matrix: host-platform: - linux-x64 + - linux-aarch64 + - win-x64 target-device: - gpu build-mode: From d167a5536815d393c6d2dcaa1838965daa133bfa Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 06:43:01 +0000 Subject: [PATCH 12/65] fix CIBW_BUILD for windows --- .github/actions/build/action.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index bf121c1b6..bf095323b 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -32,10 +32,14 @@ runs: - name: Build cuda.core wheel uses: pypa/cibuildwheel@v2.22.0 env: - CIBW_BUILD: ${{ (inputs.python-version == '3.12' && 'cp312-manylinux*') || - (inputs.python-version == '3.11' && 'cp311-manylinux*') || - (inputs.python-version == '3.10' && 'cp310-manylinux*') || - (inputs.python-version == '3.9' && 'cp39-manylinux*') }} + CIBW_BUILD: ${{ (inputs.python-version == '3.12' && inputs.host-platform == 'linux*' && 'cp312-manylinux*') || + (inputs.python-version == '3.11' && inputs.host-platform == 'linux*' && 'cp311-manylinux*') || + (inputs.python-version == '3.10' && inputs.host-platform == 'linux*' && 'cp310-manylinux*') || + (inputs.python-version == '3.9' && inputs.hott-platform == 'linux*' && 'cp39-manylinux*') || + (inputs.python-version == '3.12' && inputs.host-platform == 'win*' && 'cp312-win*') || + (inputs.python-version == '3.11' && inputs.host-platform == 'win*' && 'cp311-win*') || + (inputs.python-version == '3.10' && inputs.host-platform == 'win*' && 'cp310-win*') || + (inputs.python-version == '3.9' && inputs.hott-platform == 'win*' && 'cp39-win*') }} CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6" CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair From 9cd08038c960e0dd649cf2af59f9c6f3e968de60 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 06:45:45 +0000 Subject: [PATCH 13/65] fix typo --- .github/actions/build/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index bf095323b..1ab7ce0ed 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -35,11 +35,11 @@ runs: CIBW_BUILD: ${{ (inputs.python-version == '3.12' && inputs.host-platform == 'linux*' && 'cp312-manylinux*') || (inputs.python-version == '3.11' && inputs.host-platform == 'linux*' && 'cp311-manylinux*') || (inputs.python-version == '3.10' && inputs.host-platform == 'linux*' && 'cp310-manylinux*') || - (inputs.python-version == '3.9' && inputs.hott-platform == 'linux*' && 'cp39-manylinux*') || + (inputs.python-version == '3.9' && inputs.host-platform == 'linux*' && 'cp39-manylinux*') || (inputs.python-version == '3.12' && inputs.host-platform == 'win*' && 'cp312-win*') || (inputs.python-version == '3.11' && inputs.host-platform == 'win*' && 'cp311-win*') || (inputs.python-version == '3.10' && inputs.host-platform == 'win*' && 'cp310-win*') || - (inputs.python-version == '3.9' && inputs.hott-platform == 'win*' && 'cp39-win*') }} + (inputs.python-version == '3.9' && inputs.host-platform == 'win*' && 'cp39-win*') }} CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6" CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair From 261b1e5e197ae7b33f731ddf21cbb7c8391bf3cd Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 06:53:26 +0000 Subject: [PATCH 14/65] no quotes for wildcard matching --- .github/actions/build/action.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 1ab7ce0ed..4b891977b 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -32,14 +32,14 @@ runs: - name: Build cuda.core wheel uses: pypa/cibuildwheel@v2.22.0 env: - CIBW_BUILD: ${{ (inputs.python-version == '3.12' && inputs.host-platform == 'linux*' && 'cp312-manylinux*') || - (inputs.python-version == '3.11' && inputs.host-platform == 'linux*' && 'cp311-manylinux*') || - (inputs.python-version == '3.10' && inputs.host-platform == 'linux*' && 'cp310-manylinux*') || - (inputs.python-version == '3.9' && inputs.host-platform == 'linux*' && 'cp39-manylinux*') || - (inputs.python-version == '3.12' && inputs.host-platform == 'win*' && 'cp312-win*') || - (inputs.python-version == '3.11' && inputs.host-platform == 'win*' && 'cp311-win*') || - (inputs.python-version == '3.10' && inputs.host-platform == 'win*' && 'cp310-win*') || - (inputs.python-version == '3.9' && inputs.host-platform == 'win*' && 'cp39-win*') }} + CIBW_BUILD: ${{ (inputs.python-version == '3.12' && inputs.host-platform == linux* && 'cp312-manylinux*') || + (inputs.python-version == '3.11' && inputs.host-platform == linux* && 'cp311-manylinux*') || + (inputs.python-version == '3.10' && inputs.host-platform == linux* && 'cp310-manylinux*') || + (inputs.python-version == '3.9' && inputs.host-platform == linux* && 'cp39-manylinux*') || + (inputs.python-version == '3.12' && inputs.host-platform == win* && 'cp312-win*') || + (inputs.python-version == '3.11' && inputs.host-platform == win* && 'cp311-win*') || + (inputs.python-version == '3.10' && inputs.host-platform == win* && 'cp310-win*') || + (inputs.python-version == '3.9' && inputs.host-platform == win* && 'cp39-win*') }} CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6" CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair From 9134f898953a02b32f14d32d04d912e6ae67d7d7 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 06:58:42 +0000 Subject: [PATCH 15/65] move CIBW_BUILD logic to script --- .github/actions/build/action.yml | 9 +-------- .github/actions/setup/action.yml | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 4b891977b..42d47f541 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -32,14 +32,7 @@ runs: - name: Build cuda.core wheel uses: pypa/cibuildwheel@v2.22.0 env: - CIBW_BUILD: ${{ (inputs.python-version == '3.12' && inputs.host-platform == linux* && 'cp312-manylinux*') || - (inputs.python-version == '3.11' && inputs.host-platform == linux* && 'cp311-manylinux*') || - (inputs.python-version == '3.10' && inputs.host-platform == linux* && 'cp310-manylinux*') || - (inputs.python-version == '3.9' && inputs.host-platform == linux* && 'cp39-manylinux*') || - (inputs.python-version == '3.12' && inputs.host-platform == win* && 'cp312-win*') || - (inputs.python-version == '3.11' && inputs.host-platform == win* && 'cp311-win*') || - (inputs.python-version == '3.10' && inputs.host-platform == win* && 'cp310-win*') || - (inputs.python-version == '3.9' && inputs.host-platform == win* && 'cp39-win*') }} + CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6" CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a48d2f51e..29fbe3581 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -46,6 +46,26 @@ runs: exit 1 fi + if [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.host-platform }}" == linux* ]]; then + CIBW_BUILD='cp312-manylinux*' + elif [[ "${{ inputs.python-version }}" == "3.11" && "${{ inputs.host-platform }}" == linux* ]]; then + CIBW_BUILD='cp311-manylinux*' + elif [[ "${{ inputs.python-version }}" == "3.10" && "${{ inputs.host-platform }}" == linux* ]]; then + CIBW_BUILD='cp310-manylinux*' + elif [[ "${{ inputs.python-version }}" == "3.9" && "${{ inputs.host-platform }}" == linux* ]]; then + CIBW_BUILD='cp39-manylinux*' + elif [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.host-platform }}" == win* ]]; then + CIBW_BUILD='cp312-win*' + elif [[ "${{ inputs.python-version }}" == "3.11" && "${{ inputs.host-platform }}" == win* ]]; then + CIBW_BUILD='cp311-win*' + elif [[ "${{ inputs.python-version }}" == "3.10" && "${{ inputs.host-platform }}" == win* ]]; then + CIBW_BUILD='cp310-win*' + elif [[ "${{ inputs.python-version }}" == "3.9" && "${{ inputs.host-platform }}" == win* ]]; then + CIBW_BUILD='cp39-win*' + else + exit 1 + fi + BUILD_MODE="${{ inputs.build-mode }}" if [[ ("${BUILD_MODE}" == "") || ("${BUILD_MODE}" == "release") ]]; then # We upload release versions in the default folder. @@ -63,3 +83,4 @@ runs: echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV + echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV From 762ad764666b6f14d6fc4e184787ae9ce8b2bc38 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 07:01:49 +0000 Subject: [PATCH 16/65] fix win runner name --- .github/workflows/gh-build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml index eae1c25a1..867cffde0 100644 --- a/.github/workflows/gh-build-and-test.yml +++ b/.github/workflows/gh-build-and-test.yml @@ -29,7 +29,7 @@ jobs: target-device: ${{ inputs.target-device }} runs-on: ${{ (inputs.host-platform == 'linux-x64' && 'linux-amd64-cpu8') || (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') || - (inputs.host-platform == 'win-x64' && 'linux-amd64-cpu8') }} + (inputs.host-platform == 'win-x64' && 'windows-amd64-cpu8') }} build-type: ${{ inputs.build-type }} host-platform: ${{ inputs.host-platform }} build-mode: ${{ inputs.build-mode }} From 0d3c32bd3b4ec8fdfecadebf6b570d6e63c0804c Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 07:40:53 +0000 Subject: [PATCH 17/65] try to find where pwsh is --- .github/actions/setup/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 29fbe3581..dde7862b8 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -74,6 +74,12 @@ runs: PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" fi + if [[ "${{ inputs.host-platform }}" == win* ]]; then + ls $PROGRAMFILES + ls $PROGRAMFILES\PowerShell + ls C:\Windows\System32\ + fi + echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${{ inputs.python-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV From 37006e80c295ed7b055f482bc74aafed98c06add Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 07:46:06 +0000 Subject: [PATCH 18/65] try to escape --- .github/actions/setup/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index dde7862b8..77670f6ee 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -75,9 +75,9 @@ runs: fi if [[ "${{ inputs.host-platform }}" == win* ]]; then - ls $PROGRAMFILES - ls $PROGRAMFILES\PowerShell - ls C:\Windows\System32\ + ls "${PROGRAMFILES}" + ls "${PROGRAMFILES}\\PowerShell" + ls "C:\\Windows\\System32\\" fi echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV From 0ab50675bb16e7e168af76b786cf438a726671ec Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 07:51:22 +0000 Subject: [PATCH 19/65] continue hunting.. --- .github/actions/setup/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 77670f6ee..44095f588 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -75,8 +75,7 @@ runs: fi if [[ "${{ inputs.host-platform }}" == win* ]]; then - ls "${PROGRAMFILES}" - ls "${PROGRAMFILES}\\PowerShell" + ls "${PROGRAMFILES}\\WindowsPowerShell" ls "C:\\Windows\\System32\\" fi From d42f8ddd4bf93ea1180c77a8e87cc680fb1d1591 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 08:02:48 +0000 Subject: [PATCH 20/65] try to overwrite shell --- .github/actions/build/action.yml | 1 + .github/actions/setup/action.yml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 42d47f541..6ba09c46f 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -40,6 +40,7 @@ runs: with: package-dir: ./cuda_core/ output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + shell: ${{ WAR_SHELL }} #- name: Display structure of the bindings artifacts folder (post build) # shell: bash --noprofile --norc -xeuo pipefail {0} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 44095f588..3f9c146fc 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -75,8 +75,10 @@ runs: fi if [[ "${{ inputs.host-platform }}" == win* ]]; then - ls "${PROGRAMFILES}\\WindowsPowerShell" - ls "C:\\Windows\\System32\\" + ls "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\" + echo "WAR_SHELL=powershell" >> $GITHUB_ENV + else + echo "WAR_SHELL=bash" >> $GITHUB_ENV fi echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV From 55c03705f0276775fe31032efd44f6d02e509582 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 15:37:59 +0000 Subject: [PATCH 21/65] try to install ps --- .github/actions/build/action.yml | 1 - .github/actions/setup/action.yml | 17 ++++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 6ba09c46f..42d47f541 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -40,7 +40,6 @@ runs: with: package-dir: ./cuda_core/ output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - shell: ${{ WAR_SHELL }} #- name: Display structure of the bindings artifacts folder (post build) # shell: bash --noprofile --norc -xeuo pipefail {0} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 3f9c146fc..38aab81f7 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -26,6 +26,16 @@ inputs: runs: using: composite steps: + - name: Install PowerShell + if: ${{ startsWith(inputs.host-platform, 'win') }} + env: + # doesn't seem there's an easy way to avoid hard-coding it? + PS_MSI_URL: https://github.com/PowerShell/PowerShell/releases/download/v7.4.6/PowerShell-7.4.6-win-x64.msi + run: | + Invoke-WebRequest -Uri "$env:PS_MSI_URL" -OutFile "ps_installer.msi" + Start-Process msiexec.exe -Wait -Verbose -ArgumentList '/i "ps_installer.msi" /qn REGISTER_MANIFEST=1' + echo "$env:ProgramFiles\\PowerShell\\7" >> $env:GITHUB_PATH + - name: Set REPO_DIR and Dump environment shell: bash --noprofile --norc -xeuo pipefail {0} run: | @@ -74,13 +84,6 @@ runs: PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" fi - if [[ "${{ inputs.host-platform }}" == win* ]]; then - ls "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\" - echo "WAR_SHELL=powershell" >> $GITHUB_ENV - else - echo "WAR_SHELL=bash" >> $GITHUB_ENV - fi - echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${{ inputs.python-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV From f3356f44cf9788944e7267e16b36ec847b5ab232 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 15:42:49 +0000 Subject: [PATCH 22/65] be explicit about shell (why?) --- .github/actions/setup/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 38aab81f7..ecc5eb56d 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -28,6 +28,7 @@ runs: steps: - name: Install PowerShell if: ${{ startsWith(inputs.host-platform, 'win') }} + shell: powershell env: # doesn't seem there's an easy way to avoid hard-coding it? PS_MSI_URL: https://github.com/PowerShell/PowerShell/releases/download/v7.4.6/PowerShell-7.4.6-win-x64.msi From 5480f10c8de0d818ff933c9c50390c1da4e318f5 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 15:56:52 +0000 Subject: [PATCH 23/65] only build for win 64 bits --- .github/actions/setup/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index ecc5eb56d..739977713 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -66,13 +66,13 @@ runs: elif [[ "${{ inputs.python-version }}" == "3.9" && "${{ inputs.host-platform }}" == linux* ]]; then CIBW_BUILD='cp39-manylinux*' elif [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.host-platform }}" == win* ]]; then - CIBW_BUILD='cp312-win*' + CIBW_BUILD='cp312-win_amd64' elif [[ "${{ inputs.python-version }}" == "3.11" && "${{ inputs.host-platform }}" == win* ]]; then - CIBW_BUILD='cp311-win*' + CIBW_BUILD='cp311-win_amd64' elif [[ "${{ inputs.python-version }}" == "3.10" && "${{ inputs.host-platform }}" == win* ]]; then - CIBW_BUILD='cp310-win*' + CIBW_BUILD='cp310-win_amd64' elif [[ "${{ inputs.python-version }}" == "3.9" && "${{ inputs.host-platform }}" == win* ]]; then - CIBW_BUILD='cp39-win*' + CIBW_BUILD='cp39-win_amd64' else exit 1 fi From 4b1a43188860f9bb26bcd2ad13a9c0e9de5b6fa4 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 16:26:04 +0000 Subject: [PATCH 24/65] try to install msvc --- .github/actions/build/action.yml | 9 --------- .github/actions/setup/action.yml | 14 +++++++++++++- .github/workflows/gh-build.yml | 1 - 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 42d47f541..a61b4222f 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -16,19 +16,10 @@ inputs: upload-enabled: required: true type: boolean - python-version: - required: true - type: string runs: using: composite steps: - - name: Set up Python ${{ inputs.python-version }} - id: setup-python - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - - name: Build cuda.core wheel uses: pypa/cibuildwheel@v2.22.0 env: diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 739977713..94c6c0a5e 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -26,7 +26,19 @@ inputs: runs: using: composite steps: - - name: Install PowerShell + - name: Set up Python ${{ inputs.python-version }} + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Set up MSVC + if: ${{ startsWith(inputs.host-platform, 'win') }} + uses: ilammy/msvc-dev-cmd@v1 + with: + vsversion: 2017 + + - name: Set up new PowerShell (pwsh) if: ${{ startsWith(inputs.host-platform, 'win') }} shell: powershell env: diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 70f7772d6..f0f419c6c 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -67,4 +67,3 @@ jobs: target-device: "${{ inputs.target-device }}" host-platform: ${{ inputs.host-platform }} upload-enabled: ${{ inputs.upload-enabled }} - python-version: ${{ inputs.python-version }} From 7044c8b7b265dab24bcc76dd04fe75de04ac7222 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 17:33:51 +0000 Subject: [PATCH 25/65] install msvc ourselves --- .github/actions/setup/action.yml | 42 ++++++++++++--- continuous_integration/scripts/install-vs.ps1 | 53 +++++++++++++++++++ 2 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 continuous_integration/scripts/install-vs.ps1 diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 94c6c0a5e..b7cdddca1 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -32,12 +32,6 @@ runs: with: python-version: ${{ inputs.python-version }} - - name: Set up MSVC - if: ${{ startsWith(inputs.host-platform, 'win') }} - uses: ilammy/msvc-dev-cmd@v1 - with: - vsversion: 2017 - - name: Set up new PowerShell (pwsh) if: ${{ startsWith(inputs.host-platform, 'win') }} shell: powershell @@ -49,6 +43,42 @@ runs: Start-Process msiexec.exe -Wait -Verbose -ArgumentList '/i "ps_installer.msi" /qn REGISTER_MANIFEST=1' echo "$env:ProgramFiles\\PowerShell\\7" >> $env:GITHUB_PATH + - name: Install MSVC + if: ${{ startsWith(inputs.host-platform, 'win') }} + shell: pwsh + env: + run: | + # Download & install + .\\continuous_integration\\scripts\\install-vs.ps1 -msvcVersion 15 -clVersion 14.16 + + # Verify + # Note: this assumes vs2017, e.g. see _find_vc2017(): + # https://github.com/pypa/setuptools/blob/9692cde009af4651819d18a1e839d3b6e3fcd77d/setuptools/_distutils/_msvccompiler.py#L67 + $vsPath = & "${env:ProgramFiles(x86)}\\Microsoft Visual Studio\\Installer\\vswhere.exe" ` + -latest ` + -products * ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -property installationPath + $clPath = Join-Path $vsPath "VC\\Tools\\MSVC\\*\\bin\\Hostx64\\x64\\cl.exe" + $clPath = (Get-ChildItem $clPath).FullName + echo "found cl.exe: $clPath" + # For some reason below just doesn't work in the CI... + Start-Process -NoNewWindow -RedirectStandardError cl.out -FilePath "$clPath" + $CL_VERSION_STRING = & type cl.out + if (($CL_VERSION_STRING -join " ") -match "Version (\\d+\\.\\d+)\\.\\d+") { + $CL_VERSION = $matches[1] + echo "Detected cl.exe version: $CL_VERSION" + } + + # Delete temporary items created by installers and such + Remove-Item -Recurse $ENV:TEMP\* + + - name: Set up MSVC + if: ${{ startsWith(inputs.host-platform, 'win') }} + uses: ilammy/msvc-dev-cmd@v1 + with: + vsversion: 2017 + - name: Set REPO_DIR and Dump environment shell: bash --noprofile --norc -xeuo pipefail {0} run: | diff --git a/continuous_integration/scripts/install-vs.ps1 b/continuous_integration/scripts/install-vs.ps1 new file mode 100644 index 000000000..3b70b3034 --- /dev/null +++ b/continuous_integration/scripts/install-vs.ps1 @@ -0,0 +1,53 @@ +# from https://github.com/rapidsai/devcontainers/blob/branch-25.02/windows/image/installers/install-vs.ps1 + +Param( + [Parameter(Mandatory=$true)] + [ValidateSet('15', '16', '17')] + [string] + $msvcVersion, + [Parameter(Mandatory=$false)] + [string] + $clVersion +) + +$msvcPath = "C:\msbuild\$msvcVersion" + +$vsComponentsMap = @{ + "15" = "Microsoft.VisualStudio.Component.Windows10SDK.17763" + "16" = "Microsoft.VisualStudio.Component.Windows10SDK.20348" + "17" = "Microsoft.VisualStudio.Component.Windows10SDK.20348" + "14.14" = "Microsoft.VisualStudio.Component.VC.Tools.14.14" + "14.15" = "Microsoft.VisualStudio.Component.VC.Tools.14.15" + "14.16" = "Microsoft.VisualStudio.Component.VC.Tools.14.16" + "14.27" = "Microsoft.VisualStudio.Component.VC.14.27.x86.x64" + "14.28" = "Microsoft.VisualStudio.Component.VC.14.28.x86.x64" + "14.29" = "Microsoft.VisualStudio.Component.VC.14.29.x86.x64" + "14.34" = "Microsoft.VisualStudio.Component.VC.14.34.17.4.x86.x64" + "14.35" = "Microsoft.VisualStudio.Component.VC.14.35.17.5.x86.x64" + "14.36" = "Microsoft.VisualStudio.Component.VC.14.36.17.6.x86.x64" + "14.37" = "Microsoft.VisualStudio.Component.VC.14.37.17.7.x86.x64" + "14.38" = "Microsoft.VisualStudio.Component.VC.14.38.17.8.x86.x64" + "14.39" = "Microsoft.VisualStudio.Component.VC.14.39.17.9.x86.x64" + "latest" = "Microsoft.VisualStudio.Component.VC.Tools.x86.x64" +} + +# Always install/update core VC tools +$vsComponent = $vsComponentsMap[$msvcVersion] +$vsComponentString = "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 --add $vsComponent" + +if ($clVersion) { + $clComponent = $vsComponentsMap[$clVersion] + $vsComponentString = "$vsComponentString --add $clComponent" +} + +# 14.16 alone does not install build tools +if ($clVersion -eq "14.16") { + $clComponent = $vsComponentsMap["14.15"] + $vsComponentString = "$vsComponentString --add $clComponent" +} + +Invoke-WebRequest -Uri "https://aka.ms/vs/$msvcVersion/release/vs_buildtools.exe" -UseBasicParsing -OutFile .\vs_buildtools.exe +Write-Output "Installing components: $vsComponentString" +Start-Process -NoNewWindow -PassThru -Wait -FilePath .\vs_buildtools.exe -ArgumentList "install --installWhileDownloading --installPath $msvcPath --wait --norestart --nocache --quiet $vsComponentString" + +Remove-Item .\vs_buildtools.exe From 9fbdff24c9ae826bc763dc149eeb1b723a7f8c9f Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 17:35:28 +0000 Subject: [PATCH 26/65] fix typo --- .github/actions/setup/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index b7cdddca1..aa77bff4c 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -46,7 +46,6 @@ runs: - name: Install MSVC if: ${{ startsWith(inputs.host-platform, 'win') }} shell: pwsh - env: run: | # Download & install .\\continuous_integration\\scripts\\install-vs.ps1 -msvcVersion 15 -clVersion 14.16 From 8f77f1752164eb3f5b8ee4c96ac74d286c169d3e Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 17:57:35 +0000 Subject: [PATCH 27/65] skip custom cl ver check --- .github/actions/setup/action.yml | 17 +++++++++-------- .github/workflows/gh-build.yml | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index aa77bff4c..715ed65fb 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -61,21 +61,22 @@ runs: $clPath = Join-Path $vsPath "VC\\Tools\\MSVC\\*\\bin\\Hostx64\\x64\\cl.exe" $clPath = (Get-ChildItem $clPath).FullName echo "found cl.exe: $clPath" - # For some reason below just doesn't work in the CI... - Start-Process -NoNewWindow -RedirectStandardError cl.out -FilePath "$clPath" - $CL_VERSION_STRING = & type cl.out - if (($CL_VERSION_STRING -join " ") -match "Version (\\d+\\.\\d+)\\.\\d+") { - $CL_VERSION = $matches[1] - echo "Detected cl.exe version: $CL_VERSION" - } + # # For some reason below just doesn't work in the CI... + # Start-Process -NoNewWindow -RedirectStandardError cl.out -FilePath "$clPath" + # $CL_VERSION_STRING = & type cl.out + # if (($CL_VERSION_STRING -join " ") -match "Version (\\d+\\.\\d+)\\.\\d+") { + # $CL_VERSION = $matches[1] + # echo "Detected cl.exe version: $CL_VERSION" + # } # Delete temporary items created by installers and such - Remove-Item -Recurse $ENV:TEMP\* + Remove-Item -Recurse "${env:TEMP}\\*" - name: Set up MSVC if: ${{ startsWith(inputs.host-platform, 'win') }} uses: ilammy/msvc-dev-cmd@v1 with: + toolset: 14.16 vsversion: 2017 - name: Set REPO_DIR and Dump environment diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index f0f419c6c..494517692 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -49,7 +49,7 @@ jobs: with: fetch-depth: 0 - - name: Setup environment variables + - name: Set up build environment uses: ./.github/actions/setup with: client-repo: ${{ inputs.client-repo }} From 29e9e2ed27f4f57aa6fd5c6dedb40ebc0629e313 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 18:29:02 +0000 Subject: [PATCH 28/65] install to standard location --- continuous_integration/scripts/install-vs.ps1 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/continuous_integration/scripts/install-vs.ps1 b/continuous_integration/scripts/install-vs.ps1 index 3b70b3034..3b10a6472 100644 --- a/continuous_integration/scripts/install-vs.ps1 +++ b/continuous_integration/scripts/install-vs.ps1 @@ -1,4 +1,4 @@ -# from https://github.com/rapidsai/devcontainers/blob/branch-25.02/windows/image/installers/install-vs.ps1 +# Modified from https://github.com/rapidsai/devcontainers/blob/branch-25.02/windows/image/installers/install-vs.ps1 Param( [Parameter(Mandatory=$true)] @@ -10,8 +10,6 @@ Param( $clVersion ) -$msvcPath = "C:\msbuild\$msvcVersion" - $vsComponentsMap = @{ "15" = "Microsoft.VisualStudio.Component.Windows10SDK.17763" "16" = "Microsoft.VisualStudio.Component.Windows10SDK.20348" @@ -48,6 +46,6 @@ if ($clVersion -eq "14.16") { Invoke-WebRequest -Uri "https://aka.ms/vs/$msvcVersion/release/vs_buildtools.exe" -UseBasicParsing -OutFile .\vs_buildtools.exe Write-Output "Installing components: $vsComponentString" -Start-Process -NoNewWindow -PassThru -Wait -FilePath .\vs_buildtools.exe -ArgumentList "install --installWhileDownloading --installPath $msvcPath --wait --norestart --nocache --quiet $vsComponentString" +Start-Process -NoNewWindow -PassThru -Wait -FilePath .\vs_buildtools.exe -ArgumentList "install --installWhileDownloading --wait --norestart --nocache --quiet $vsComponentString" Remove-Item .\vs_buildtools.exe From e2cffcea8d260934a1caed3b3b15657f4e740b59 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 19:24:16 +0000 Subject: [PATCH 29/65] try to locate Python include path --- .github/actions/build/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index a61b4222f..cfe8f66c8 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -28,6 +28,9 @@ runs: CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 + CIBW_BEFORE_BUILD_WINDOWS: > + # ensure Python.h & co can be found + python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE with: package-dir: ./cuda_core/ output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} From a7157b5608583f5996cecf48617507c24b7a1619 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 20:02:26 +0000 Subject: [PATCH 30/65] switch to public windows runner for now --- .github/actions/build/action.yml | 6 +- .github/actions/setup/action.yml | 78 ++++++++++++------------- .github/workflows/gh-build-and-test.yml | 3 +- 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index cfe8f66c8..e93d11140 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -28,9 +28,9 @@ runs: CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 - CIBW_BEFORE_BUILD_WINDOWS: > - # ensure Python.h & co can be found - python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE + # # ensure Python.h & co can be found + # CIBW_BEFORE_BUILD_WINDOWS: > + # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE with: package-dir: ./cuda_core/ output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 715ed65fb..6f9ed1dc8 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -32,52 +32,52 @@ runs: with: python-version: ${{ inputs.python-version }} - - name: Set up new PowerShell (pwsh) - if: ${{ startsWith(inputs.host-platform, 'win') }} - shell: powershell - env: - # doesn't seem there's an easy way to avoid hard-coding it? - PS_MSI_URL: https://github.com/PowerShell/PowerShell/releases/download/v7.4.6/PowerShell-7.4.6-win-x64.msi - run: | - Invoke-WebRequest -Uri "$env:PS_MSI_URL" -OutFile "ps_installer.msi" - Start-Process msiexec.exe -Wait -Verbose -ArgumentList '/i "ps_installer.msi" /qn REGISTER_MANIFEST=1' - echo "$env:ProgramFiles\\PowerShell\\7" >> $env:GITHUB_PATH + # - name: Set up new PowerShell (pwsh) + # if: ${{ startsWith(inputs.host-platform, 'win') }} + # shell: powershell + # env: + # # doesn't seem there's an easy way to avoid hard-coding it? + # PS_MSI_URL: https://github.com/PowerShell/PowerShell/releases/download/v7.4.6/PowerShell-7.4.6-win-x64.msi + # run: | + # Invoke-WebRequest -Uri "$env:PS_MSI_URL" -OutFile "ps_installer.msi" + # Start-Process msiexec.exe -Wait -Verbose -ArgumentList '/i "ps_installer.msi" /qn REGISTER_MANIFEST=1' + # echo "$env:ProgramFiles\\PowerShell\\7" >> $env:GITHUB_PATH - - name: Install MSVC - if: ${{ startsWith(inputs.host-platform, 'win') }} - shell: pwsh - run: | - # Download & install - .\\continuous_integration\\scripts\\install-vs.ps1 -msvcVersion 15 -clVersion 14.16 + # - name: Install MSVC + # if: ${{ startsWith(inputs.host-platform, 'win') }} + # shell: pwsh + # run: | + # # Download & install + # .\\continuous_integration\\scripts\\install-vs.ps1 -msvcVersion 15 -clVersion 14.16 - # Verify - # Note: this assumes vs2017, e.g. see _find_vc2017(): - # https://github.com/pypa/setuptools/blob/9692cde009af4651819d18a1e839d3b6e3fcd77d/setuptools/_distutils/_msvccompiler.py#L67 - $vsPath = & "${env:ProgramFiles(x86)}\\Microsoft Visual Studio\\Installer\\vswhere.exe" ` - -latest ` - -products * ` - -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` - -property installationPath - $clPath = Join-Path $vsPath "VC\\Tools\\MSVC\\*\\bin\\Hostx64\\x64\\cl.exe" - $clPath = (Get-ChildItem $clPath).FullName - echo "found cl.exe: $clPath" - # # For some reason below just doesn't work in the CI... - # Start-Process -NoNewWindow -RedirectStandardError cl.out -FilePath "$clPath" - # $CL_VERSION_STRING = & type cl.out - # if (($CL_VERSION_STRING -join " ") -match "Version (\\d+\\.\\d+)\\.\\d+") { - # $CL_VERSION = $matches[1] - # echo "Detected cl.exe version: $CL_VERSION" - # } + # # Verify + # # Note: this assumes vs2017, e.g. see _find_vc2017(): + # # https://github.com/pypa/setuptools/blob/9692cde009af4651819d18a1e839d3b6e3fcd77d/setuptools/_distutils/_msvccompiler.py#L67 + # $vsPath = & "${env:ProgramFiles(x86)}\\Microsoft Visual Studio\\Installer\\vswhere.exe" ` + # -latest ` + # -products * ` + # -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + # -property installationPath + # $clPath = Join-Path $vsPath "VC\\Tools\\MSVC\\*\\bin\\Hostx64\\x64\\cl.exe" + # $clPath = (Get-ChildItem $clPath).FullName + # echo "found cl.exe: $clPath" + # # # For some reason below just doesn't work in the CI... + # # Start-Process -NoNewWindow -RedirectStandardError cl.out -FilePath "$clPath" + # # $CL_VERSION_STRING = & type cl.out + # # if (($CL_VERSION_STRING -join " ") -match "Version (\\d+\\.\\d+)\\.\\d+") { + # # $CL_VERSION = $matches[1] + # # echo "Detected cl.exe version: $CL_VERSION" + # # } - # Delete temporary items created by installers and such - Remove-Item -Recurse "${env:TEMP}\\*" + # # Delete temporary items created by installers and such + # Remove-Item -Recurse "${env:TEMP}\\*" - name: Set up MSVC if: ${{ startsWith(inputs.host-platform, 'win') }} uses: ilammy/msvc-dev-cmd@v1 - with: - toolset: 14.16 - vsversion: 2017 + # with: + # toolset: 14.16 + # vsversion: 2017 - name: Set REPO_DIR and Dump environment shell: bash --noprofile --norc -xeuo pipefail {0} diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml index 867cffde0..8d1c1b97d 100644 --- a/.github/workflows/gh-build-and-test.yml +++ b/.github/workflows/gh-build-and-test.yml @@ -29,7 +29,8 @@ jobs: target-device: ${{ inputs.target-device }} runs-on: ${{ (inputs.host-platform == 'linux-x64' && 'linux-amd64-cpu8') || (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') || - (inputs.host-platform == 'win-x64' && 'windows-amd64-cpu8') }} + (inputs.host-platform == 'win-x64' && 'windows-2019') }} + # (inputs.host-platform == 'win-x64' && 'windows-amd64-cpu8') }} build-type: ${{ inputs.build-type }} host-platform: ${{ inputs.host-platform }} build-mode: ${{ inputs.build-mode }} From 965b23f04e8eb8b88a26f251742ccbc0cd2bb852 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 20:18:46 +0000 Subject: [PATCH 31/65] windows image does not have sudo --- .github/actions/build/action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index e93d11140..0ed4ab192 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -50,7 +50,12 @@ runs: - name: List the cuda.core artifacts directory shell: bash --noprofile --norc -xeuo pipefail {0} run: | - sudo chown -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }} + if [[ "${{ inputs.host-platform }}" == win* ]]; then + export CHOWN=chown + else + export CHOWN="sudo chown" + fi + $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }} ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - name: Upload cuda.core build artifacts From 8ae7bd2e633ede890ba536063321539dcbfcf921 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 20:51:04 +0000 Subject: [PATCH 32/65] pwd on Windows Bash does not use Windows path format --- .github/actions/setup/action.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 6f9ed1dc8..3fd9e06d7 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -82,7 +82,12 @@ runs: - name: Set REPO_DIR and Dump environment shell: bash --noprofile --norc -xeuo pipefail {0} run: | - echo "REPO_DIR=$(pwd)" >> $GITHUB_ENV + if [[ "${{ inputs.host-platform }}" == linux* ]]; then + echo "REPO_DIR=$(pwd)" >> $GITHUB_ENV + else + PWD=$(pwd) + echo "REPO_DIR=$(cygpath -w $PWD)" >> $GITHUB_ENV + fi env - name: Set environment variables @@ -127,9 +132,9 @@ runs: PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" fi - echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV + echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${{ inputs.python-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV - echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV + echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV From 2175ccbcc2db39621497b471dd69d3a8900bd6dd Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 20:59:34 +0000 Subject: [PATCH 33/65] cover all Python versions! --- .github/workflows/ci-gh.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index cf1683257..43cb16bbe 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -27,8 +27,11 @@ jobs: upload-enabled: - false python-version: - #TODO cover the whole python and cuda matrix - 3.12 + - 3.11 + - 3.10 + - 3.9 + # TODO cover the whole cuda matrix uses: ./.github/workflows/gh-build-and-test.yml with: From 50eb0e732d94f43ce5e424d94ad434357bad89bd Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 21:06:35 +0000 Subject: [PATCH 34/65] add quotes --- .github/workflows/ci-gh.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index 43cb16bbe..62bb036cd 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -27,10 +27,10 @@ jobs: upload-enabled: - false python-version: - - 3.12 - - 3.11 - - 3.10 - - 3.9 + - "3.12" + - "3.11" + - "3.10" + - "3.9" # TODO cover the whole cuda matrix uses: ./.github/workflows/gh-build-and-test.yml From b0046ac5cc6d9b42ff381b07c8dbdab3640e3d4d Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 21:13:17 +0000 Subject: [PATCH 35/65] align the python version installed via GHA vs used at build time --- .github/actions/build/action.yml | 2 +- .github/actions/setup/action.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 0ed4ab192..e42be1c9c 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -24,7 +24,7 @@ runs: uses: pypa/cibuildwheel@v2.22.0 env: CIBW_BUILD: ${{ env.CIBW_BUILD }} - CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6" + CIBW_PROJECT_REQUIRES_PYTHON: ${{ env.CIBW_PROJECT_REQUIRES_PYTHON }} CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 3fd9e06d7..95953799e 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -142,3 +142,4 @@ runs: echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV + echo "CIBW_PROJECT_REQUIRES_PYTHON=${{ inputs.python-version }}" >> $GITHUB_ENV From 493266739f19ff99a99855526f14a02fdedc921b Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 21:16:49 +0000 Subject: [PATCH 36/65] fix constraint syntax --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 95953799e..f3624dd0c 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -142,4 +142,4 @@ runs: echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV - echo "CIBW_PROJECT_REQUIRES_PYTHON=${{ inputs.python-version }}" >> $GITHUB_ENV + echo "CIBW_PROJECT_REQUIRES_PYTHON=\"==${{ inputs.python-version }}\"" >> $GITHUB_ENV From 7cb84c3a147dfbd5a60923bb4db4045020ca0395 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 21:30:04 +0000 Subject: [PATCH 37/65] check if setup-python is causing interference --- .github/actions/build/action.yml | 2 +- .github/actions/setup/action.yml | 28 +++++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index e42be1c9c..1b58ba6c2 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -24,7 +24,7 @@ runs: uses: pypa/cibuildwheel@v2.22.0 env: CIBW_BUILD: ${{ env.CIBW_BUILD }} - CIBW_PROJECT_REQUIRES_PYTHON: ${{ env.CIBW_PROJECT_REQUIRES_PYTHON }} + #CIBW_PROJECT_REQUIRES_PYTHON: ${{ env.CIBW_PROJECT_REQUIRES_PYTHON }} CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index f3624dd0c..7b90e8ee7 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -26,11 +26,11 @@ inputs: runs: using: composite steps: - - name: Set up Python ${{ inputs.python-version }} - id: setup-python - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} + # - name: Set up Python ${{ inputs.python-version }} + # id: setup-python + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ inputs.python-version }} # - name: Set up new PowerShell (pwsh) # if: ${{ startsWith(inputs.host-platform, 'win') }} @@ -79,15 +79,9 @@ runs: # toolset: 14.16 # vsversion: 2017 - - name: Set REPO_DIR and Dump environment + - name: Dump environment shell: bash --noprofile --norc -xeuo pipefail {0} run: | - if [[ "${{ inputs.host-platform }}" == linux* ]]; then - echo "REPO_DIR=$(pwd)" >> $GITHUB_ENV - else - PWD=$(pwd) - echo "REPO_DIR=$(cygpath -w $PWD)" >> $GITHUB_ENV - fi env - name: Set environment variables @@ -132,6 +126,14 @@ runs: PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" fi + if [[ "${{ inputs.host-platform }}" == linux* ]]; then + REPO_DIR=$(pwd)" + else + PWD=$(pwd) + REPO_DIR=$(cygpath -w $PWD)" + fi + + echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${{ inputs.python-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV @@ -142,4 +144,4 @@ runs: echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV - echo "CIBW_PROJECT_REQUIRES_PYTHON=\"==${{ inputs.python-version }}\"" >> $GITHUB_ENV + #echo "CIBW_PROJECT_REQUIRES_PYTHON=\"==${{ inputs.python-version }}\"" >> $GITHUB_ENV From d97fbddb7a583e70b95182065f18292b962c2c7a Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 21:35:20 +0000 Subject: [PATCH 38/65] fix typo --- .github/actions/setup/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 7b90e8ee7..bf7022f3d 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -127,10 +127,10 @@ runs: fi if [[ "${{ inputs.host-platform }}" == linux* ]]; then - REPO_DIR=$(pwd)" + REPO_DIR=$(pwd) else PWD=$(pwd) - REPO_DIR=$(cygpath -w $PWD)" + REPO_DIR=$(cygpath -w $PWD) fi echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV From 59e4e0a464498346c5f4ee311969e2c20cd5711a Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 22:32:54 +0000 Subject: [PATCH 39/65] apply a WAR on Linux --- .github/actions/setup/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index bf7022f3d..d8e29bccf 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -128,6 +128,9 @@ runs: if [[ "${{ inputs.host-platform }}" == linux* ]]; then REPO_DIR=$(pwd) + # WAR: setup-python is not relocatable... + # see https://github.com/actions/setup-python/issues/871 + echo "LD_LIBRARY_PATH=/opt/hostedtoolcache/Python/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV else PWD=$(pwd) REPO_DIR=$(cygpath -w $PWD) @@ -144,4 +147,3 @@ runs: echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV - #echo "CIBW_PROJECT_REQUIRES_PYTHON=\"==${{ inputs.python-version }}\"" >> $GITHUB_ENV From 2ab0877b0ceb0d670ac0ebc5732b33da01a9c5fd Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 22:35:51 +0000 Subject: [PATCH 40/65] fix unbound var --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index d8e29bccf..876803897 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -130,7 +130,7 @@ runs: REPO_DIR=$(pwd) # WAR: setup-python is not relocatable... # see https://github.com/actions/setup-python/issues/871 - echo "LD_LIBRARY_PATH=/opt/hostedtoolcache/Python/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/opt/hostedtoolcache/Python/lib" >> $GITHUB_ENV else PWD=$(pwd) REPO_DIR=$(cygpath -w $PWD) From cac9de8b6c485d25898e9702271e69501a05bd31 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 22:48:03 +0000 Subject: [PATCH 41/65] detect Python path after it's installed (by CIBW) --- .github/actions/build/action.yml | 4 ++++ .github/actions/setup/action.yml | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 1b58ba6c2..84b36683e 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -28,6 +28,10 @@ runs: CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 + # WAR: setup-python is not relocatable... + # see https://github.com/actions/setup-python/issues/871 + CIBW_BEFORE_BUILD_LINUX: > + export LD_LIBRARY_PATH="$(dirname $(which python))/../lib:${LD_LIBRARY_PATH:-}" # # ensure Python.h & co can be found # CIBW_BEFORE_BUILD_WINDOWS: > # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 876803897..09ab9f139 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -128,9 +128,6 @@ runs: if [[ "${{ inputs.host-platform }}" == linux* ]]; then REPO_DIR=$(pwd) - # WAR: setup-python is not relocatable... - # see https://github.com/actions/setup-python/issues/871 - echo "LD_LIBRARY_PATH=/opt/hostedtoolcache/Python/lib" >> $GITHUB_ENV else PWD=$(pwd) REPO_DIR=$(cygpath -w $PWD) From a3ef630d7bc627aa53cf4f56875dfd12752d7f36 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 22:52:39 +0000 Subject: [PATCH 42/65] try CIBW_BEFORE_ALL_LINUX --- .github/actions/build/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 84b36683e..925dd2caa 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -30,7 +30,7 @@ runs: CIBW_BUILD_VERBOSITY: 1 # WAR: setup-python is not relocatable... # see https://github.com/actions/setup-python/issues/871 - CIBW_BEFORE_BUILD_LINUX: > + CIBW_BEFORE_ALL_LINUX: > export LD_LIBRARY_PATH="$(dirname $(which python))/../lib:${LD_LIBRARY_PATH:-}" # # ensure Python.h & co can be found # CIBW_BEFORE_BUILD_WINDOWS: > From 26af84d139605a86aa1276e156abaa3262e56573 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Thu, 5 Dec 2024 23:36:55 +0000 Subject: [PATCH 43/65] try to restore the pre-py-span setup... --- .github/actions/build/action.yml | 4 ---- .github/actions/setup/action.yml | 13 ++++++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 925dd2caa..1b58ba6c2 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -28,10 +28,6 @@ runs: CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 - # WAR: setup-python is not relocatable... - # see https://github.com/actions/setup-python/issues/871 - CIBW_BEFORE_ALL_LINUX: > - export LD_LIBRARY_PATH="$(dirname $(which python))/../lib:${LD_LIBRARY_PATH:-}" # # ensure Python.h & co can be found # CIBW_BEFORE_BUILD_WINDOWS: > # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 09ab9f139..d478718c3 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -26,11 +26,14 @@ inputs: runs: using: composite steps: - # - name: Set up Python ${{ inputs.python-version }} - # id: setup-python - # uses: actions/setup-python@v5 - # with: - # python-version: ${{ inputs.python-version }} + # WAR: setup-python is not relocatable... + # see https://github.com/actions/setup-python/issues/871 + - name: Set up Python ${{ inputs.python-version }} + if: ${{ startsWith(inputs.host-platform, 'linux') }} + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: "3.12" # - name: Set up new PowerShell (pwsh) # if: ${{ startsWith(inputs.host-platform, 'win') }} From cdbdfb5598f4e747bf93b27426abc99aff9f4a3d Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 00:08:59 +0000 Subject: [PATCH 44/65] reduce build matrix to experiment with cuda.bindings builds --- .github/actions/build/action.yml | 46 ++++++++++++++++++------- .github/actions/setup/action.yml | 30 +++++++++++++++- .github/workflows/ci-gh.yml | 16 +++++---- .github/workflows/gh-build-and-test.yml | 5 +++ .github/workflows/gh-build.yml | 6 +++- 5 files changed, 82 insertions(+), 21 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 1b58ba6c2..befef9b2c 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -24,7 +24,6 @@ runs: uses: pypa/cibuildwheel@v2.22.0 env: CIBW_BUILD: ${{ env.CIBW_BUILD }} - #CIBW_PROJECT_REQUIRES_PYTHON: ${{ env.CIBW_PROJECT_REQUIRES_PYTHON }} CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 @@ -35,18 +34,6 @@ runs: package-dir: ./cuda_core/ output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }} - #- name: Display structure of the bindings artifacts folder (post build) - # shell: bash --noprofile --norc -xeuo pipefail {0} - # run: | - # sudo chown -R $(whoami) ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - # ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - - #- name: Upload bindings build artifacts - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} - # path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} - - name: List the cuda.core artifacts directory shell: bash --noprofile --norc -xeuo pipefail {0} run: | @@ -65,3 +52,36 @@ runs: path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl if-no-files-found: error overwrite: 'true' + + - name: Build cuda.bindings wheel + uses: pypa/cibuildwheel@v2.22.0 + env: + CIBW_BUILD: ${{ env.CIBW_BUILD }} + CIBW_ARCHS_LINUX: "native" + CIBW_REPAIR_WHEEL_COMMAND: "" # no repair + CIBW_BUILD_VERBOSITY: 1 + # # ensure Python.h & co can be found + # CIBW_BEFORE_BUILD_WINDOWS: > + # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE + with: + package-dir: ./cuda_bindings/ + output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + + - name: List the cuda.bindings artifacts directory + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + if [[ "${{ inputs.host-platform }}" == win* ]]; then + export CHOWN=chown + else + export CHOWN="sudo chown" + fi + $CHOWN -R $(whoami) ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }} + + - name: Upload cuda.bindings build artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }} + path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl + if-no-files-found: error + overwrite: 'true' diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index d478718c3..49fec033a 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -22,6 +22,9 @@ inputs: python-version: required: true type: string + cuda-version: + required: true + type: string runs: using: composite @@ -87,6 +90,29 @@ runs: run: | env + - name: Get CUDA headers + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + CUDA_PATH="cuda-${{ inputs.cuda-version }}" + mkdir -p $CUDA_PATH/include + + git clone -b "cuda-${{ inputs.cuda-version }}" https://gitlab.com/nvidia/headers/cuda-individual/cudart.git + rm -rf cudart/.git + mv cudart/* $CUDA_PATH/include + rmdir cudart + + git clone -b "cuda-${{ inputs.cuda-version }}" https://gitlab.com/nvidia/headers/cuda-individual/nvcc.git + rm -rf nvcc/.git + mv nvcc/* $CUDA_PATH/include + rmdir nvcc + + git clone -b "cuda-${{ inputs.cuda-version }}" https://gitlab.com/nvidia/headers/cuda-individual/nvrtc.git + rm -rf nvrtc/.git + mv nvrtc/* $CUDA_PATH/include + rmdir nvrtc + + echo "CUDA_PATH=$CUDA_PATH" >> $GITHUB_ENV + - name: Set environment variables shell: bash --noprofile --norc -xeuo pipefail {0} run: | @@ -136,10 +162,12 @@ runs: REPO_DIR=$(cygpath -w $PWD) fi + echo "PARALLEL_LEVEL=${nproc}" >> $GITHUB_ENV echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV - echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${{ inputs.python-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV + echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${{ inputs.python-version }}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV + echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index 62bb036cd..fd702f5a3 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -18,8 +18,8 @@ jobs: matrix: host-platform: - linux-x64 - - linux-aarch64 - - win-x64 + #- linux-aarch64 + #- win-x64 target-device: - gpu build-mode: @@ -28,10 +28,13 @@ jobs: - false python-version: - "3.12" - - "3.11" - - "3.10" - - "3.9" - # TODO cover the whole cuda matrix + #- "3.11" + #- "3.10" + #- "3.9" + cuda-version: + # Note: this is for build-time only; the test-time matrix needs to be + # defined separately. + - "12.6.2" uses: ./.github/workflows/gh-build-and-test.yml with: @@ -41,4 +44,5 @@ jobs: build-type: ci upload-enabled: ${{ matrix.upload-enabled }} python-version: ${{ matrix.python-version }} + cuda-version: ${{ matrix.cuda-version }} secrets: inherit diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml index 8d1c1b97d..a9a711d4f 100644 --- a/.github/workflows/gh-build-and-test.yml +++ b/.github/workflows/gh-build-and-test.yml @@ -19,6 +19,10 @@ on: python-version: type: string required: true + cuda-version: + type: string + required: true + jobs: build: if: ${{ github.repository_owner == 'nvidia' }} @@ -36,5 +40,6 @@ jobs: build-mode: ${{ inputs.build-mode }} upload-enabled: ${{ inputs.upload-enabled }} python-version: ${{ inputs.python-version }} + cuda-version: ${{ inputs.cuda-version }} dependencies-file: "" secrets: inherit diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 494517692..7a9f03ce3 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -32,10 +32,13 @@ on: python-version: required: true type: string + cuda-version: + required: true + type: string jobs: build: - name: Build (${{ inputs.host-platform }}, ${{ inputs.target-device }}, ${{ inputs.build-type }}, build-mode=${{ inputs.build-mode }}, Python "${{ inputs.python-version }}") + name: Build (${{ inputs.host-platform }}, ${{ inputs.build-type }}, ${{ inputs.build-mode }}, Python "${{ inputs.python-version }}") permissions: id-token: write # This is required for configure-aws-credentials @@ -59,6 +62,7 @@ jobs: build-mode: ${{ inputs.build-mode }} upload-enabled: ${{ inputs.upload-enabled }} python-version: ${{ inputs.python-version }} + cuda-version: ${{ inputs.cuda-version }} - name: Call build action uses: ./.github/actions/build From 8a61031631904847f6633cfb1e38756ffe0a0f86 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 00:11:06 +0000 Subject: [PATCH 45/65] fix parenthesis --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 49fec033a..75d746953 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -162,7 +162,7 @@ runs: REPO_DIR=$(cygpath -w $PWD) fi - echo "PARALLEL_LEVEL=${nproc}" >> $GITHUB_ENV + echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${{ inputs.python-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV From 35b88265853449ec52cf860711df68ab5094d111 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 00:16:13 +0000 Subject: [PATCH 46/65] use abs path --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 75d746953..7f4f5e9b5 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -111,7 +111,7 @@ runs: mv nvrtc/* $CUDA_PATH/include rmdir nvrtc - echo "CUDA_PATH=$CUDA_PATH" >> $GITHUB_ENV + echo "CUDA_PATH=$(realpath $CUDA_PATH)" >> $GITHUB_ENV - name: Set environment variables shell: bash --noprofile --norc -xeuo pipefail {0} From 8e42dc41e5caacfa4d17359edc15d2402dc28695 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 00:24:30 +0000 Subject: [PATCH 47/65] defer CUDA_PATH setting --- .github/actions/build/action.yml | 2 ++ .github/actions/setup/action.yml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index befef9b2c..7a0ae5e25 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -60,6 +60,8 @@ runs: CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 + CIBW_BEFORE_BUILD_LINUX: > + export CUDA_PATH="$(realpath ./cuda-*)" # TODO: use cuda-version here? # # ensure Python.h & co can be found # CIBW_BEFORE_BUILD_WINDOWS: > # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 7f4f5e9b5..484af8204 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -111,7 +111,8 @@ runs: mv nvrtc/* $CUDA_PATH/include rmdir nvrtc - echo "CUDA_PATH=$(realpath $CUDA_PATH)" >> $GITHUB_ENV + # Note: the headers will be copied into the cibuildwheel manylinux container, + # so setting the CUDA_PATH env var here is meaningless. - name: Set environment variables shell: bash --noprofile --norc -xeuo pipefail {0} From 1588e575e4f004285c3fa5e038e8fb40f8576bdc Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 01:07:32 +0000 Subject: [PATCH 48/65] use CIBW_ENVIRONMENT to pass env var --- .github/actions/build/action.yml | 4 ++-- .github/actions/setup/action.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 7a0ae5e25..fb5236a4a 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -60,8 +60,8 @@ runs: CIBW_ARCHS_LINUX: "native" CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 - CIBW_BEFORE_BUILD_LINUX: > - export CUDA_PATH="$(realpath ./cuda-*)" # TODO: use cuda-version here? + CIBW_ENVIRONMENT: > + CUDA_PATH="$(realpath ./cuda_toolkit)" # # ensure Python.h & co can be found # CIBW_BEFORE_BUILD_WINDOWS: > # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 484af8204..b33bb4d62 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -93,7 +93,7 @@ runs: - name: Get CUDA headers shell: bash --noprofile --norc -xeuo pipefail {0} run: | - CUDA_PATH="cuda-${{ inputs.cuda-version }}" + CUDA_PATH="./cuda_toolkit" mkdir -p $CUDA_PATH/include git clone -b "cuda-${{ inputs.cuda-version }}" https://gitlab.com/nvidia/headers/cuda-individual/cudart.git From e13854df68de7c133bc16d9835e89f1d77e72409 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 01:43:43 +0000 Subject: [PATCH 49/65] fetch cuda-profiler-api headers --- .github/actions/setup/action.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index b33bb4d62..a3b6bfbab 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -111,6 +111,33 @@ runs: mv nvrtc/* $CUDA_PATH/include rmdir nvrtc + # cuda-profiler-api aren't updated on Gitlab... + # TODO: consider applying the same approach for all components? The binary archives (redist) are + # guaranteed to be updated as part of the release posting. + CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/" + CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json" + if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then + CTK_SUBDIR="linux-x86_64" + elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then + CTK_SUBDIR="linux-sbsa" + elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then + CTK_SUBDIR="windows-x86_64" + else + exit 1 + fi + CTK_COMPONENT="cuda_profiler_api" + CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL | + python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")" + CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}" + wget $CTK_COMPONENT_URL + CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)" + if [[ "${{ inputs.host-platform }}" == linux* ]]; then + tar -xvf $CTK_COMPONENT_COMPONENT_FILENAME -C $CUDA_PATH --strip-components=1 + else + exit 1 # FIXME + fi + rm $CTK_COMPONENT_COMPONENT_FILENAME + # Note: the headers will be copied into the cibuildwheel manylinux container, # so setting the CUDA_PATH env var here is meaningless. From 4f735090b4864794e62cd4788534ad7fdd609a1e Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 02:00:31 +0000 Subject: [PATCH 50/65] only rely on redist --- .github/actions/setup/action.yml | 54 +++++++++++++------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a3b6bfbab..d067dabb2 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -90,30 +90,13 @@ runs: run: | env - - name: Get CUDA headers + - name: Get CUDA components shell: bash --noprofile --norc -xeuo pipefail {0} run: | CUDA_PATH="./cuda_toolkit" - mkdir -p $CUDA_PATH/include - - git clone -b "cuda-${{ inputs.cuda-version }}" https://gitlab.com/nvidia/headers/cuda-individual/cudart.git - rm -rf cudart/.git - mv cudart/* $CUDA_PATH/include - rmdir cudart - - git clone -b "cuda-${{ inputs.cuda-version }}" https://gitlab.com/nvidia/headers/cuda-individual/nvcc.git - rm -rf nvcc/.git - mv nvcc/* $CUDA_PATH/include - rmdir nvcc - - git clone -b "cuda-${{ inputs.cuda-version }}" https://gitlab.com/nvidia/headers/cuda-individual/nvrtc.git - rm -rf nvrtc/.git - mv nvrtc/* $CUDA_PATH/include - rmdir nvrtc - - # cuda-profiler-api aren't updated on Gitlab... - # TODO: consider applying the same approach for all components? The binary archives (redist) are - # guaranteed to be updated as part of the release posting. + mkdir $CUDA_PATH + + # The binary archives (redist) are guaranteed to be updated as part of the release posting. CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/" CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json" if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then @@ -125,18 +108,23 @@ runs: else exit 1 fi - CTK_COMPONENT="cuda_profiler_api" - CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL | - python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")" - CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}" - wget $CTK_COMPONENT_URL - CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)" - if [[ "${{ inputs.host-platform }}" == linux* ]]; then - tar -xvf $CTK_COMPONENT_COMPONENT_FILENAME -C $CUDA_PATH --strip-components=1 - else - exit 1 # FIXME - fi - rm $CTK_COMPONENT_COMPONENT_FILENAME + function populate_cuda_path() { + # take the component name as a argument + CTK_COMPONENT=$1 + CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL | + python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")" + CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}" + wget $CTK_COMPONENT_URL + CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)" + tar -xvf $CTK_COMPONENT_COMPONENT_FILENAME -C $CUDA_PATH --strip-components=1 + rm $CTK_COMPONENT_COMPONENT_FILENAME + } + + # Get headers and shared libraries in place + populate_cuda_path cuda_nvcc + populate_cuda_path cuda_cudart + populate_cuda_path cuda_nvrtc + populate_cuda_path cuda_profiler_api # Note: the headers will be copied into the cibuildwheel manylinux container, # so setting the CUDA_PATH env var here is meaningless. From 128ab1241caddf80765d2358111c221bcbcf3b17 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 02:08:35 +0000 Subject: [PATCH 51/65] allow wheel repair to fix the triplet tags --- .github/actions/build/action.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index fb5236a4a..c53970700 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -25,7 +25,6 @@ runs: env: CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_ARCHS_LINUX: "native" - CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 # # ensure Python.h & co can be found # CIBW_BEFORE_BUILD_WINDOWS: > @@ -58,7 +57,6 @@ runs: env: CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_ARCHS_LINUX: "native" - CIBW_REPAIR_WHEEL_COMMAND: "" # no repair CIBW_BUILD_VERBOSITY: 1 CIBW_ENVIRONMENT: > CUDA_PATH="$(realpath ./cuda_toolkit)" From 6087a54def96154f39c7c3413e69d149bfa98537 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 02:23:16 +0000 Subject: [PATCH 52/65] restore full build matrix! --- .github/workflows/ci-gh.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index fd702f5a3..1975c3b52 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -18,8 +18,8 @@ jobs: matrix: host-platform: - linux-x64 - #- linux-aarch64 - #- win-x64 + - linux-aarch64 + - win-x64 target-device: - gpu build-mode: @@ -28,9 +28,9 @@ jobs: - false python-version: - "3.12" - #- "3.11" - #- "3.10" - #- "3.9" + - "3.11" + - "3.10" + - "3.9" cuda-version: # Note: this is for build-time only; the test-time matrix needs to be # defined separately. From b05e95dad7386e18eccef1f2e6378636ba6459e9 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 02:45:19 +0000 Subject: [PATCH 53/65] fix wget on Windows; pass PARALLEL_LEVEL to CIBW --- .github/actions/build/action.yml | 1 + .github/actions/setup/action.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index c53970700..abad93c26 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -60,6 +60,7 @@ runs: CIBW_BUILD_VERBOSITY: 1 CIBW_ENVIRONMENT: > CUDA_PATH="$(realpath ./cuda_toolkit)" + PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }} # # ensure Python.h & co can be found # CIBW_BEFORE_BUILD_WINDOWS: > # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index d067dabb2..656858e71 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -105,9 +105,11 @@ runs: CTK_SUBDIR="linux-sbsa" elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then CTK_SUBDIR="windows-x86_64" - else - exit 1 + apt update; apt install wget fi + function extractor() { + tar -xvf $1 -C $CUDA_PATH --strip-components=1 + } function populate_cuda_path() { # take the component name as a argument CTK_COMPONENT=$1 @@ -116,7 +118,7 @@ runs: CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}" wget $CTK_COMPONENT_URL CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)" - tar -xvf $CTK_COMPONENT_COMPONENT_FILENAME -C $CUDA_PATH --strip-components=1 + extractor $CTK_COMPONENT_COMPONENT_FILENAME rm $CTK_COMPONENT_COMPONENT_FILENAME } @@ -180,14 +182,12 @@ runs: echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV + echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${{ inputs.python-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${{ inputs.python-version }}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV - echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV - echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV - echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV From 0c91ed8c9106701685f24917f53e510961f375bb Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 02:57:50 +0000 Subject: [PATCH 54/65] switch from wget to curl --- .github/actions/setup/action.yml | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 656858e71..264cd1108 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -105,21 +105,23 @@ runs: CTK_SUBDIR="linux-sbsa" elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then CTK_SUBDIR="windows-x86_64" - apt update; apt install wget fi - function extractor() { - tar -xvf $1 -C $CUDA_PATH --strip-components=1 - } function populate_cuda_path() { - # take the component name as a argument - CTK_COMPONENT=$1 - CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL | - python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")" - CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}" - wget $CTK_COMPONENT_URL - CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)" - extractor $CTK_COMPONENT_COMPONENT_FILENAME - rm $CTK_COMPONENT_COMPONENT_FILENAME + # take the component name as a argument + function download() { + curl -kLSs $1 -o $2 + } + function extract() { + tar -xvf $1 -C $CUDA_PATH --strip-components=1 + } + CTK_COMPONENT=$1 + CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL | + python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")" + CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}" + CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)" + download $CTK_COMPONENT_URL $CTK_COMPONENT_COMPONENT_FILENAME + extract $CTK_COMPONENT_COMPONENT_FILENAME + rm $CTK_COMPONENT_COMPONENT_FILENAME } # Get headers and shared libraries in place From 33f35c336cb1720eb4cbeb27fe4676cb50456fdd Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 03:15:59 +0000 Subject: [PATCH 55/65] windows needs unzip not tar --- .github/actions/setup/action.yml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 264cd1108..a5a0ab296 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -99,21 +99,29 @@ runs: # The binary archives (redist) are guaranteed to be updated as part of the release posting. CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/" CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json" - if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then - CTK_SUBDIR="linux-x86_64" - elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then - CTK_SUBDIR="linux-sbsa" + if [[ "${{ inputs.host-platform }}" == linux* ]]; then + if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then + CTK_SUBDIR="linux-x86_64" + elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then + CTK_SUBDIR="linux-sbsa" + fi + function extract() { + tar -xvf $1 -C $CUDA_PATH --strip-components=1 + } elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then CTK_SUBDIR="windows-x86_64" + function extract() { + _TEMP_DIR_=$(mktemp -d) + unzip $1 -d $_TEMP_DIR_ + mv $_TEMP_DIR_/*/* $CUDA_PATH + rmdir $_TEMP_DIR_/* $_TEMP_DIR_ + } fi function populate_cuda_path() { # take the component name as a argument function download() { curl -kLSs $1 -o $2 } - function extract() { - tar -xvf $1 -C $CUDA_PATH --strip-components=1 - } CTK_COMPONENT=$1 CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL | python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")" From 4b3341a80bcf9b1145e69bdcd20084b4abf05cdb Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 03:23:09 +0000 Subject: [PATCH 56/65] mv -> rsync --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a5a0ab296..0ad9cadf6 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -113,7 +113,7 @@ runs: function extract() { _TEMP_DIR_=$(mktemp -d) unzip $1 -d $_TEMP_DIR_ - mv $_TEMP_DIR_/*/* $CUDA_PATH + rsync -avh --remove-source-files $_TEMP_DIR_/*/* $CUDA_PATH rmdir $_TEMP_DIR_/* $_TEMP_DIR_ } fi From 3503dae21a8b524fe754acd5d252b9025f975bc2 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 03:32:35 +0000 Subject: [PATCH 57/65] git bash has no wget or rsync... --- .github/actions/setup/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 0ad9cadf6..c61c87da5 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -113,8 +113,8 @@ runs: function extract() { _TEMP_DIR_=$(mktemp -d) unzip $1 -d $_TEMP_DIR_ - rsync -avh --remove-source-files $_TEMP_DIR_/*/* $CUDA_PATH - rmdir $_TEMP_DIR_/* $_TEMP_DIR_ + cp -r $_TEMP_DIR_/* $CUDA_PATH + rm -r $_TEMP_DIR_ } fi function populate_cuda_path() { From dc12840f8243c2621d9910508444a3e8230aca39 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 03:39:39 +0000 Subject: [PATCH 58/65] ensure win-style path on win --- .github/actions/build/action.yml | 5 ++++- .github/actions/setup/action.yml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index abad93c26..91c47ca0e 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -58,9 +58,12 @@ runs: CIBW_BUILD: ${{ env.CIBW_BUILD }} CIBW_ARCHS_LINUX: "native" CIBW_BUILD_VERBOSITY: 1 - CIBW_ENVIRONMENT: > + CIBW_ENVIRONMENT_LINUX: > CUDA_PATH="$(realpath ./cuda_toolkit)" PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }} + CIBW_ENVIRONMENT_WINDOWS: > + CUDA_HOME="$(cygpath -w ./cuda_toolkit)" + PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }} # # ensure Python.h & co can be found # CIBW_BEFORE_BUILD_WINDOWS: > # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index c61c87da5..6c21e221f 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -114,7 +114,7 @@ runs: _TEMP_DIR_=$(mktemp -d) unzip $1 -d $_TEMP_DIR_ cp -r $_TEMP_DIR_/* $CUDA_PATH - rm -r $_TEMP_DIR_ + rm -rf $_TEMP_DIR_ } fi function populate_cuda_path() { From c55bdc55ad610fd16c4387d70318bca3c2a7e65f Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 03:43:56 +0000 Subject: [PATCH 59/65] convert abs path --- .github/actions/build/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 91c47ca0e..fc9ff6014 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -62,7 +62,7 @@ runs: CUDA_PATH="$(realpath ./cuda_toolkit)" PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }} CIBW_ENVIRONMENT_WINDOWS: > - CUDA_HOME="$(cygpath -w ./cuda_toolkit)" + CUDA_HOME="$(cygpath -w $(realpath ./cuda_toolkit))" PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }} # # ensure Python.h & co can be found # CIBW_BEFORE_BUILD_WINDOWS: > From bdb35999c1e7b2fd09e7bf9337ba1a9c2fab9aed Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 03:54:22 +0000 Subject: [PATCH 60/65] debug --- .github/actions/setup/action.yml | 52 +++++++++++++++----------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 6c21e221f..67b51a2d2 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -138,6 +138,8 @@ runs: populate_cuda_path cuda_nvrtc populate_cuda_path cuda_profiler_api + ls -l $CUDA_PATH + tree $CUDA_PATH # Note: the headers will be copied into the cibuildwheel manylinux container, # so setting the CUDA_PATH env var here is meaningless. @@ -151,28 +153,31 @@ runs: TARGET_PLATFORM='linux-aarch64' elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then TARGET_PLATFORM='win-64' - else - exit 1 fi - if [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.host-platform }}" == linux* ]]; then - CIBW_BUILD='cp312-manylinux*' - elif [[ "${{ inputs.python-version }}" == "3.11" && "${{ inputs.host-platform }}" == linux* ]]; then - CIBW_BUILD='cp311-manylinux*' - elif [[ "${{ inputs.python-version }}" == "3.10" && "${{ inputs.host-platform }}" == linux* ]]; then - CIBW_BUILD='cp310-manylinux*' - elif [[ "${{ inputs.python-version }}" == "3.9" && "${{ inputs.host-platform }}" == linux* ]]; then - CIBW_BUILD='cp39-manylinux*' - elif [[ "${{ inputs.python-version }}" == "3.12" && "${{ inputs.host-platform }}" == win* ]]; then - CIBW_BUILD='cp312-win_amd64' - elif [[ "${{ inputs.python-version }}" == "3.11" && "${{ inputs.host-platform }}" == win* ]]; then - CIBW_BUILD='cp311-win_amd64' - elif [[ "${{ inputs.python-version }}" == "3.10" && "${{ inputs.host-platform }}" == win* ]]; then - CIBW_BUILD='cp310-win_amd64' - elif [[ "${{ inputs.python-version }}" == "3.9" && "${{ inputs.host-platform }}" == win* ]]; then - CIBW_BUILD='cp39-win_amd64' - else - exit 1 + if [[ "${{ inputs.host-platform }}" == linux* ]]; then + if [[ "${{ inputs.python-version }}" == "3.12" ]]; then + CIBW_BUILD='cp312-manylinux*' + elif [[ "${{ inputs.python-version }}" == "3.11" ]]; then + CIBW_BUILD='cp311-manylinux*' + elif [[ "${{ inputs.python-version }}" == "3.10" ]]; then + CIBW_BUILD='cp310-manylinux*' + elif [[ "${{ inputs.python-version }}" == "3.9" ]]; then + CIBW_BUILD='cp39-manylinux*' + fi + REPO_DIR=$(pwd) + elif [[ "${{ inputs.host-platform }}" == win* ]]; then + if [[ "${{ inputs.python-version }}" == "3.12" ]]; then + CIBW_BUILD='cp312-win_amd64' + elif [[ "${{ inputs.python-version }}" == "3.11" ]]; then + CIBW_BUILD='cp311-win_amd64' + elif [[ "${{ inputs.python-version }}" == "3.10" ]]; then + CIBW_BUILD='cp310-win_amd64' + elif [[ "${{ inputs.python-version }}" == "3.9" ]]; then + CIBW_BUILD='cp39-win_amd64' + fi + PWD=$(pwd) + REPO_DIR=$(cygpath -w $PWD) fi BUILD_MODE="${{ inputs.build-mode }}" @@ -183,13 +188,6 @@ runs: PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" fi - if [[ "${{ inputs.host-platform }}" == linux* ]]; then - REPO_DIR=$(pwd) - else - PWD=$(pwd) - REPO_DIR=$(cygpath -w $PWD) - fi - echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV From c8fd323954f94bd8783ee82c375b3e34da1c61b5 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 03:57:29 +0000 Subject: [PATCH 61/65] another level down --- .github/actions/setup/action.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 67b51a2d2..db525c75f 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -113,7 +113,7 @@ runs: function extract() { _TEMP_DIR_=$(mktemp -d) unzip $1 -d $_TEMP_DIR_ - cp -r $_TEMP_DIR_/* $CUDA_PATH + cp -r $_TEMP_DIR_/*/* $CUDA_PATH rm -rf $_TEMP_DIR_ } fi @@ -137,9 +137,8 @@ runs: populate_cuda_path cuda_cudart populate_cuda_path cuda_nvrtc populate_cuda_path cuda_profiler_api - ls -l $CUDA_PATH - tree $CUDA_PATH + # Note: the headers will be copied into the cibuildwheel manylinux container, # so setting the CUDA_PATH env var here is meaningless. From 0d554f0945b1bd1b7797170ece2a13765ab6dbe8 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 04:23:29 +0000 Subject: [PATCH 62/65] check if it is a race condition on win --- .github/actions/build/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index fc9ff6014..e1552ae83 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -63,7 +63,7 @@ runs: PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }} CIBW_ENVIRONMENT_WINDOWS: > CUDA_HOME="$(cygpath -w $(realpath ./cuda_toolkit))" - PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }} + # PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }} # # ensure Python.h & co can be found # CIBW_BEFORE_BUILD_WINDOWS: > # python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE From 19445d58143f0dc58193a39080c5808a4e52d220 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 15:41:32 -0500 Subject: [PATCH 63/65] clean up unused (but still relevant) code --- .github/actions/setup/action.yml | 43 -------------------------------- 1 file changed, 43 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index db525c75f..63c120877 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -38,52 +38,9 @@ runs: with: python-version: "3.12" - # - name: Set up new PowerShell (pwsh) - # if: ${{ startsWith(inputs.host-platform, 'win') }} - # shell: powershell - # env: - # # doesn't seem there's an easy way to avoid hard-coding it? - # PS_MSI_URL: https://github.com/PowerShell/PowerShell/releases/download/v7.4.6/PowerShell-7.4.6-win-x64.msi - # run: | - # Invoke-WebRequest -Uri "$env:PS_MSI_URL" -OutFile "ps_installer.msi" - # Start-Process msiexec.exe -Wait -Verbose -ArgumentList '/i "ps_installer.msi" /qn REGISTER_MANIFEST=1' - # echo "$env:ProgramFiles\\PowerShell\\7" >> $env:GITHUB_PATH - - # - name: Install MSVC - # if: ${{ startsWith(inputs.host-platform, 'win') }} - # shell: pwsh - # run: | - # # Download & install - # .\\continuous_integration\\scripts\\install-vs.ps1 -msvcVersion 15 -clVersion 14.16 - - # # Verify - # # Note: this assumes vs2017, e.g. see _find_vc2017(): - # # https://github.com/pypa/setuptools/blob/9692cde009af4651819d18a1e839d3b6e3fcd77d/setuptools/_distutils/_msvccompiler.py#L67 - # $vsPath = & "${env:ProgramFiles(x86)}\\Microsoft Visual Studio\\Installer\\vswhere.exe" ` - # -latest ` - # -products * ` - # -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` - # -property installationPath - # $clPath = Join-Path $vsPath "VC\\Tools\\MSVC\\*\\bin\\Hostx64\\x64\\cl.exe" - # $clPath = (Get-ChildItem $clPath).FullName - # echo "found cl.exe: $clPath" - # # # For some reason below just doesn't work in the CI... - # # Start-Process -NoNewWindow -RedirectStandardError cl.out -FilePath "$clPath" - # # $CL_VERSION_STRING = & type cl.out - # # if (($CL_VERSION_STRING -join " ") -match "Version (\\d+\\.\\d+)\\.\\d+") { - # # $CL_VERSION = $matches[1] - # # echo "Detected cl.exe version: $CL_VERSION" - # # } - - # # Delete temporary items created by installers and such - # Remove-Item -Recurse "${env:TEMP}\\*" - - name: Set up MSVC if: ${{ startsWith(inputs.host-platform, 'win') }} uses: ilammy/msvc-dev-cmd@v1 - # with: - # toolset: 14.16 - # vsversion: 2017 - name: Dump environment shell: bash --noprofile --norc -xeuo pipefail {0} From 15626a8e781adf8417754556fc6fba6fd7511fa9 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 15:41:51 -0500 Subject: [PATCH 64/65] clean up unused (but still relevant) code - cont'd --- continuous_integration/scripts/install-vs.ps1 | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 continuous_integration/scripts/install-vs.ps1 diff --git a/continuous_integration/scripts/install-vs.ps1 b/continuous_integration/scripts/install-vs.ps1 deleted file mode 100644 index 3b10a6472..000000000 --- a/continuous_integration/scripts/install-vs.ps1 +++ /dev/null @@ -1,51 +0,0 @@ -# Modified from https://github.com/rapidsai/devcontainers/blob/branch-25.02/windows/image/installers/install-vs.ps1 - -Param( - [Parameter(Mandatory=$true)] - [ValidateSet('15', '16', '17')] - [string] - $msvcVersion, - [Parameter(Mandatory=$false)] - [string] - $clVersion -) - -$vsComponentsMap = @{ - "15" = "Microsoft.VisualStudio.Component.Windows10SDK.17763" - "16" = "Microsoft.VisualStudio.Component.Windows10SDK.20348" - "17" = "Microsoft.VisualStudio.Component.Windows10SDK.20348" - "14.14" = "Microsoft.VisualStudio.Component.VC.Tools.14.14" - "14.15" = "Microsoft.VisualStudio.Component.VC.Tools.14.15" - "14.16" = "Microsoft.VisualStudio.Component.VC.Tools.14.16" - "14.27" = "Microsoft.VisualStudio.Component.VC.14.27.x86.x64" - "14.28" = "Microsoft.VisualStudio.Component.VC.14.28.x86.x64" - "14.29" = "Microsoft.VisualStudio.Component.VC.14.29.x86.x64" - "14.34" = "Microsoft.VisualStudio.Component.VC.14.34.17.4.x86.x64" - "14.35" = "Microsoft.VisualStudio.Component.VC.14.35.17.5.x86.x64" - "14.36" = "Microsoft.VisualStudio.Component.VC.14.36.17.6.x86.x64" - "14.37" = "Microsoft.VisualStudio.Component.VC.14.37.17.7.x86.x64" - "14.38" = "Microsoft.VisualStudio.Component.VC.14.38.17.8.x86.x64" - "14.39" = "Microsoft.VisualStudio.Component.VC.14.39.17.9.x86.x64" - "latest" = "Microsoft.VisualStudio.Component.VC.Tools.x86.x64" -} - -# Always install/update core VC tools -$vsComponent = $vsComponentsMap[$msvcVersion] -$vsComponentString = "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 --add $vsComponent" - -if ($clVersion) { - $clComponent = $vsComponentsMap[$clVersion] - $vsComponentString = "$vsComponentString --add $clComponent" -} - -# 14.16 alone does not install build tools -if ($clVersion -eq "14.16") { - $clComponent = $vsComponentsMap["14.15"] - $vsComponentString = "$vsComponentString --add $clComponent" -} - -Invoke-WebRequest -Uri "https://aka.ms/vs/$msvcVersion/release/vs_buildtools.exe" -UseBasicParsing -OutFile .\vs_buildtools.exe -Write-Output "Installing components: $vsComponentString" -Start-Process -NoNewWindow -PassThru -Wait -FilePath .\vs_buildtools.exe -ArgumentList "install --installWhileDownloading --wait --norestart --nocache --quiet $vsComponentString" - -Remove-Item .\vs_buildtools.exe From 07b8d1d97291dbff3cf1cc6f2cde065d48ab788f Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Fri, 6 Dec 2024 21:24:22 +0000 Subject: [PATCH 65/65] consolidate with PYTHON_VERSION_FORMATTED --- .github/actions/setup/action.yml | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 63c120877..e00cf27ff 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -111,27 +111,12 @@ runs: TARGET_PLATFORM='win-64' fi + PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.') if [[ "${{ inputs.host-platform }}" == linux* ]]; then - if [[ "${{ inputs.python-version }}" == "3.12" ]]; then - CIBW_BUILD='cp312-manylinux*' - elif [[ "${{ inputs.python-version }}" == "3.11" ]]; then - CIBW_BUILD='cp311-manylinux*' - elif [[ "${{ inputs.python-version }}" == "3.10" ]]; then - CIBW_BUILD='cp310-manylinux*' - elif [[ "${{ inputs.python-version }}" == "3.9" ]]; then - CIBW_BUILD='cp39-manylinux*' - fi + CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*" REPO_DIR=$(pwd) elif [[ "${{ inputs.host-platform }}" == win* ]]; then - if [[ "${{ inputs.python-version }}" == "3.12" ]]; then - CIBW_BUILD='cp312-win_amd64' - elif [[ "${{ inputs.python-version }}" == "3.11" ]]; then - CIBW_BUILD='cp311-win_amd64' - elif [[ "${{ inputs.python-version }}" == "3.10" ]]; then - CIBW_BUILD='cp310-win_amd64' - elif [[ "${{ inputs.python-version }}" == "3.9" ]]; then - CIBW_BUILD='cp39-win_amd64' - fi + CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64" PWD=$(pwd) REPO_DIR=$(cygpath -w $PWD) fi @@ -147,9 +132,9 @@ runs: echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV - echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${{ inputs.python-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV + echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV - echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${{ inputs.python-version }}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV + echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV