From 05a08d93e625aa210bf8a6f6c955da3692ffa286 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 7 Mar 2024 17:19:57 -0800 Subject: [PATCH 1/8] Start a reusable WASI testing workflow for GH Actions --- .github/workflows/reusable-wasi.yml | 54 +++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/reusable-wasi.yml diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml new file mode 100644 index 00000000000000..e357ef74a0210d --- /dev/null +++ b/.github/workflows/reusable-wasi.yml @@ -0,0 +1,54 @@ +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + +jobs: + build_wasi_reusable: + name: 'build and test' + timeout-minutes: 60 + runs-on: ubuntu-20.04 + env: + WASI_SDK_VERSION: 20 + WASI_SDK_PATH: /opt/wasi-sdk + WASMTIME_VERSION: 18.0.2 + steps: + - uses: actions/checkout@v4 + # No problem resolver registered as one doesn't currently exist for Clang. + - name: "Install wasmtime" + uses: jcbhmr/setup-wasmtime@v2 + with: + wasmtime-version: ${{ env.WASMTIME_VERSION }} + - name: "Restore WASI SDK" + id: cache-wasi-sdk + uses: actions/cache@v4 + with: + path: ${{ env.WASI_SDK_PATH }} + key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} + - name: "Install WASI SDK" + if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + run: | + mkdir ${{ env.WASI_SDK_PATH }} && \ + curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-linux.tar.gz | \ + tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip + # XXX ccache + # XXX restore config.cache for build + # XXX restore config.cache for wasi + - name: "Install Python" + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: "Configure build Python" + run: python3 Tools/wasm/wasi.py configure-build-python -- -C --with-pydebug + - name: "Make build Python" + run: python3 Tools/wasm/wasi.py make-build-python + - name: "Configure host" + run: python3 Tools/wasm/wasi.py configure-host + - name: "Make host" + run: python3 Tools/wasm/wasi.py make-host + - name: Display build info + run: make --directory cross-build/wasm32-wasi pythoninfo + - name: "Test" + run: make --directory cross-build/wasm32-wasi test From f07c46714fe8ed957d92b2e8e6f0cfc22e03503b Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Mar 2024 12:02:39 -0800 Subject: [PATCH 2/8] Cache `config.cache` --- .github/workflows/reusable-wasi.yml | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index e357ef74a0210d..c51e9147876709 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -14,6 +14,8 @@ jobs: WASI_SDK_VERSION: 20 WASI_SDK_PATH: /opt/wasi-sdk WASMTIME_VERSION: 18.0.2 + CROSS_BUILD_PYTHON: cross-build/build + CROSS_BUILD_WASI: cross-build/wasm32-wasi steps: - uses: actions/checkout@v4 # No problem resolver registered as one doesn't currently exist for Clang. @@ -34,21 +36,30 @@ jobs: curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-linux.tar.gz | \ tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip # XXX ccache - # XXX restore config.cache for build - # XXX restore config.cache for wasi - name: "Install Python" uses: actions/setup-python@v5 with: python-version: '3.x' + - name: "Restore Python build config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: "Configure build Python" - run: python3 Tools/wasm/wasi.py configure-build-python -- -C --with-pydebug + run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" run: python3 Tools/wasm/wasi.py make-build-python + - name: "Restore host config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_WASI }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: "Configure host" - run: python3 Tools/wasm/wasi.py configure-host + # `--with-pydebug` inferred from configure-build-python + run: python3 Tools/wasm/wasi.py configure-host -- --config-cache - name: "Make host" run: python3 Tools/wasm/wasi.py make-host - - name: Display build info - run: make --directory cross-build/wasm32-wasi pythoninfo + - name: "Display build info" + run: make --directory ${{ env.CROSS_BUILD_WASI }} pythoninfo - name: "Test" - run: make --directory cross-build/wasm32-wasi test + run: make --directory ${{ env.CROSS_BUILD_WASI }} test From 7f58f8e90b8bcd1926e3c151c18c434cd16c65e2 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Mar 2024 12:04:29 -0800 Subject: [PATCH 3/8] Wire up the WASI tests --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20d1fad40ecafe..c9a5a7cf238f81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -301,6 +301,14 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build_wasi: + name: 'WASI' + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + uses: ./.github/workflows/reusable-wasi.yml + with: + config_hash: ${{ needs.check_source.outputs.config_hash }} + test_hypothesis: name: "Hypothesis tests on Ubuntu" runs-on: ubuntu-20.04 From 5eacc9a3d7d257979bc9296190c0a9a3a1235db6 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Mar 2024 12:05:21 -0800 Subject: [PATCH 4/8] Put tool versions together --- .github/workflows/reusable-wasi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index c51e9147876709..b22174f167b37e 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -11,9 +11,9 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-20.04 env: + WASMTIME_VERSION: 18.0.2 WASI_SDK_VERSION: 20 WASI_SDK_PATH: /opt/wasi-sdk - WASMTIME_VERSION: 18.0.2 CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasi steps: From 79f0c38fd45514e5cf887247598b398d0b3bd42f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Mar 2024 12:08:31 -0800 Subject: [PATCH 5/8] Add ccache --- .github/workflows/reusable-wasi.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index b22174f167b37e..b9fb09e33ad9be 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -35,7 +35,13 @@ jobs: mkdir ${{ env.WASI_SDK_PATH }} && \ curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-linux.tar.gz | \ tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip - # XXX ccache + - name: "Configure ccache action" + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: "Add ccache to PATH" + run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: "Install Python" uses: actions/setup-python@v5 with: From 690acb559c6fc81dff216e351392bfb22bba687e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Mar 2024 12:32:40 -0800 Subject: [PATCH 6/8] Make the WASI config cache key depend on the WASI SDK version --- .github/workflows/reusable-wasi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index b9fb09e33ad9be..995e669c228b5c 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -59,7 +59,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CROSS_BUILD_WASI }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }} - name: "Configure host" # `--with-pydebug` inferred from configure-build-python run: python3 Tools/wasm/wasi.py configure-host -- --config-cache From 2a3b1649e597b971dafad3c9fa75bc9119bc2c5f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Mar 2024 14:45:08 -0800 Subject: [PATCH 7/8] Add `build_wasi` to `all-required-green` --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c9a5a7cf238f81..e09b87d9d6b3f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -535,6 +535,7 @@ jobs: - build_ubuntu_ssltests - build_windows - build_windows_free_threading + - build_wasi - test_hypothesis - build_asan - cifuzz @@ -566,6 +567,7 @@ jobs: build_ubuntu, build_ubuntu_free_threading, build_ubuntu_ssltests, + build_wasi, build_windows, build_windows_free_threading, build_asan, From ee8676f24ff027def8402d915e646414de99d667 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 11 Mar 2024 10:49:49 -0700 Subject: [PATCH 8/8] Update .github/workflows/build.yml Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e09b87d9d6b3f6..ae14046935b97b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -533,9 +533,9 @@ jobs: - build_ubuntu - build_ubuntu_free_threading - build_ubuntu_ssltests + - build_wasi - build_windows - build_windows_free_threading - - build_wasi - test_hypothesis - build_asan - cifuzz