diff --git a/.github/workflows/build-toolchain.yml b/.github/workflows/build-toolchain.yml index c18685d5..c1aaa9f1 100644 --- a/.github/workflows/build-toolchain.yml +++ b/.github/workflows/build-toolchain.yml @@ -185,8 +185,12 @@ jobs: ${{ matrix.container }} docker exec swiftwasm-ci-buildbot /bin/bash -lc 'env; cp -r /source/* /home/build-user/; ./swiftwasm-build/tools/build/ci.sh ${{ matrix.scheme }}' + - name: Extract installable archive from Docker container (wasm32-unknown-wasip1) + if: ${{ matrix.container != null && matrix.scheme != 'release-6.0' && matrix.scheme != 'release-6.1' }} + run: | + docker cp swiftwasm-ci-buildbot:/home/build-user/swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-wasm32-unknown-wasip1.artifactbundle.zip . - name: Extract installable archive from Docker container (wasm32-unknown-wasi) - if: ${{ matrix.container != null }} + if: ${{ matrix.container != null && !(matrix.scheme != 'release-6.0' && matrix.scheme != 'release-6.1') }} run: | docker cp swiftwasm-ci-buildbot:/home/build-user/swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-wasm32-unknown-wasi.artifactbundle.zip . # release-6.0 doesn't have wasm32-unknown-wasip1-threads SDK support yet @@ -214,6 +218,11 @@ jobs: path: build-cache key: ${{ steps.cache_key.outputs.SCCACHE_KEY }} + - name: Upload Swift SDK artifact bundle (wasm32-unknown-wasip1) + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.scheme }}-wasm32-unknown-wasip1-artifactbundle + path: swift-wasm-${{ matrix.toolchain_channel }}-SNAPSHOT-wasm32-unknown-wasip1.artifactbundle.zip - name: Upload Swift SDK artifact bundle (wasm32-unknown-wasi) uses: actions/upload-artifact@v4 with: @@ -265,10 +274,7 @@ jobs: if: ${{ matrix.run_e2e_test }} run: | docker exec swiftwasm-ci-buildbot /bin/bash -lc \ - "./llvm-project/llvm/utils/lit/lit.py \ - --param swift-sdk=wasm32-unknown-wasi \ - --param swift-sdks-path=./swift-sdk-generator/Bundles \ - --param scheme=${{ matrix.scheme }} ./swiftwasm-build/test -vv" + "./swiftwasm-build/tools/build/run-e2e-test --scheme ${{ matrix.scheme }}" - name: Cleanup docker volume if: ${{ matrix.container != null }} diff --git a/README.md b/README.md index 7c378d8e..4dfe941c 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ $ ./tools/build/install-build-sdk.sh main # Checkout the Swift repositories in the parent directory (swiftwasm-source) and apply patches $ ./tools/git-swift-workspace --scheme main # Build the toolchain (this will take a while) -$ ./tools/build/build-toolchain.sh main +$ ./tools/build/build-toolchain main ``` See [SwiftWasm book](https://book.swiftwasm.org/contribution-guide/how-to-build-toolchain.html) for more details about dependencies you need to install and how to build on Docker. diff --git a/docs/upstreaming.md b/docs/upstreaming.md index fa0a8c42..52fdb914 100644 --- a/docs/upstreaming.md +++ b/docs/upstreaming.md @@ -65,13 +65,7 @@ If it's first time to build or patch files are changed, you need to checkout the If you want to build the whole toolchain, run the following command: ``` -./tools/build/build-toolchain.sh -``` - -If you already built the compiler and want to build only the standard library for WebAssembly, run the following command: - -``` -./tools/build/build-toolchain.sh --skip-build-host-toolchain +./tools/build/build-toolchain --scheme main ``` ### Edit diff --git a/schemes/main/build/build-foundation.sh b/schemes/main/build/build-foundation.sh deleted file mode 100755 index f34f9625..00000000 --- a/schemes/main/build/build-foundation.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -ex -DESTINATION_TOOLCHAIN=$1 -LLVM_BIN_DIR=$2 -CLANG_BIN_DIR=$3 -SWIFT_BIN_DIR=$4 -WASI_SYSROOT_PATH=$5 -TRIPLE="$6" - -SOURCE_PATH="$(cd "$(dirname $0)/../../../.." && pwd)" -BUILD_SDK_PATH="$SOURCE_PATH/build-sdk" -LIBXML2_PATH="$BUILD_SDK_PATH/libxml2-$TRIPLE" - -FOUNDATION_BUILD="$SOURCE_PATH/build/WebAssembly/foundation-$TRIPLE" - -swift_extra_flags="" -c_extra_flags="" -if [[ "$TRIPLE" == "wasm32-unknown-wasip1-threads" ]]; then - swift_extra_flags="-Xcc -matomics -Xcc -mbulk-memory -Xcc -mthread-model -Xcc posix -Xcc -pthread -Xcc -ftls-model=local-exec" - c_extra_flags="-mthread-model posix -pthread -ftls-model=local-exec" -fi - -cmake -G Ninja \ - -D CMAKE_BUILD_TYPE="Release" \ - -D CMAKE_SYSROOT="$WASI_SYSROOT_PATH" \ - -D CMAKE_Swift_COMPILER="$SWIFT_BIN_DIR/swiftc" \ - -D CMAKE_STAGING_PREFIX="$DESTINATION_TOOLCHAIN/usr" \ - -D CMAKE_SYSTEM_NAME=WASI \ - -D CMAKE_SYSTEM_PROCESSOR=wasm32 \ - -D CMAKE_C_COMPILER_TARGET="$TRIPLE" \ - -D CMAKE_CXX_COMPILER_TARGET="$TRIPLE" \ - -D CMAKE_Swift_COMPILER_TARGET="$TRIPLE" \ - -D CMAKE_C_COMPILER="$CLANG_BIN_DIR/clang" \ - -D CMAKE_CXX_COMPILER="$CLANG_BIN_DIR/clang++" \ - -D CMAKE_AR="$LLVM_BIN_DIR/llvm-ar" \ - -D CMAKE_RANLIB="$LLVM_BIN_DIR/llvm-ranlib" \ - -D LIBXML2_INCLUDE_DIR="$LIBXML2_PATH/include/libxml2" \ - -D LIBXML2_LIBRARY="$LIBXML2_PATH/lib" \ - -D BUILD_SHARED_LIBS=OFF \ - -D FOUNDATION_BUILD_TOOLS=OFF \ - -D CMAKE_Swift_COMPILER_FORCED=ON \ - -D CMAKE_C_COMPILER_FORCED=ON \ - -D CMAKE_CXX_COMPILER_FORCED=ON \ - -D CMAKE_Swift_FLAGS="-sdk $WASI_SYSROOT_PATH -resource-dir $DESTINATION_TOOLCHAIN/usr/lib/swift_static $swift_extra_flags" \ - -D CMAKE_C_FLAGS="-resource-dir $DESTINATION_TOOLCHAIN/usr/lib/swift_static/clang -B $LLVM_BIN_DIR $c_extra_flags" \ - -D CMAKE_CXX_FLAGS="-fno-exceptions -resource-dir $DESTINATION_TOOLCHAIN/usr/lib/swift_static/clang -B $LLVM_BIN_DIR $c_extra_flags" \ - -D _SwiftCollections_SourceDIR="$SOURCE_PATH/swift-collections" \ - -D _SwiftFoundation_SourceDIR="$SOURCE_PATH/swift-foundation" \ - -D _SwiftFoundationICU_SourceDIR="$SOURCE_PATH/swift-foundation-icu" \ - -D SwiftFoundation_MACRO="$SWIFT_BIN_DIR/../lib/swift/host/plugins" \ - -B "$FOUNDATION_BUILD" \ - "${SOURCE_PATH}/swift-corelibs-foundation" - -cmake --build "$FOUNDATION_BUILD" -cmake --install "$FOUNDATION_BUILD" diff --git a/schemes/main/build/build-target-toolchain.sh b/schemes/main/build/build-target-toolchain.sh index 8e53202f..77f0c901 100755 --- a/schemes/main/build/build-target-toolchain.sh +++ b/schemes/main/build/build-target-toolchain.sh @@ -40,48 +40,22 @@ build_target_toolchain() { env DESTDIR="$TRIPLE_DESTDIR" \ cmake --install "$TARGET_BUILD_ROOT/$STDLIB_PRODUCT-$HOST_SUFFIX" --prefix /usr - local swift_testing_build_dir="$TARGET_BUILD_ROOT/wasmswiftsdk-$HOST_SUFFIX/swift-testing/$TRIPLE" - # TODO: Remove this check once we build swift-testing for +threads target - if [[ -d "$swift_testing_build_dir" ]]; then - env DESTDIR="$TRIPLE_DESTDIR" \ - cmake --install "$swift_testing_build_dir" --prefix /usr - fi + env DESTDIR="$TRIPLE_DESTDIR" \ + cmake --install "$TARGET_BUILD_ROOT/wasmswiftsdk-$HOST_SUFFIX/swift-testing/$TRIPLE" --prefix /usr + env DESTDIR="$TRIPLE_DESTDIR" \ + cmake --install "$TARGET_BUILD_ROOT/wasmswiftsdk-$HOST_SUFFIX/foundation/$TRIPLE" --prefix /usr + env DESTDIR="$TRIPLE_DESTDIR" \ + cmake --install "$TARGET_BUILD_ROOT/wasmswiftsdk-$HOST_SUFFIX/xctest/$TRIPLE" --prefix /usr rm -rf "$TRIPLE_DESTDIR/usr/lib/swift_static/clang/lib/$COMPILER_RT_OS_DIR" # XXX: Is this the right way to install compiler-rt? cp -R "$TARGET_BUILD_ROOT/wasi-sysroot/$CLANG_MULTIARCH_TRIPLE/lib/$COMPILER_RT_OS_DIR" "$TRIPLE_DESTDIR/usr/lib/swift_static/clang/lib/$COMPILER_RT_OS_DIR" - # FIXME: Clang resource directory installation is not the best way currently. - # We currently have two copies of compiler headers copied from the base toolchain in - # lib/swift/clang and lib/swift_static/clang. This is because the Swift CMake build - # system installs the compiler headers from the native tools path when not building - # tools including clang compiler. This is not ideal but then where should we bring - # the compiler headers from? If we use the headers beside the base toolchain, clang - # driver will not be able to find libclang_rt.builtins-wasm32.a because it is not - # a part of the base toolchain. We need to find a better way to handle this. - local CLANG_VERSION - CLANG_VERSION="$(basename "$($CLANG_BIN_DIR/clang -print-resource-dir)")" - mkdir -p "$TRIPLE_DESTDIR/usr/lib/clang/$CLANG_VERSION/lib" - ln -sf "../../../swift_static/clang/lib/$COMPILER_RT_OS_DIR" "$TRIPLE_DESTDIR/usr/lib/clang/$CLANG_VERSION/lib/$COMPILER_RT_OS_DIR" -} - -build_target_corelibs() { - local LLVM_BIN_DIR="$1" - local CLANG_BIN_DIR="$2" - local SWIFT_BIN_DIR="$3" - local TRIPLE="$4" - local SHORT_TRIPLE="$5" - - local TRIPLE_DESTDIR="$TARGET_TOOLCHAIN_DESTDIR/$TRIPLE" - local CORELIBS_ARGS=( - "$TRIPLE_DESTDIR" - "$LLVM_BIN_DIR" - "$CLANG_BIN_DIR" - "$SWIFT_BIN_DIR" - "$WASI_SYSROOT_PATH/$SHORT_TRIPLE" - ) - "$SCHEMES_BUILD_PATH/build-foundation.sh" "${CORELIBS_ARGS[@]}" "$TRIPLE" - "$SCHEMES_BUILD_PATH/build-xctest.sh" "${CORELIBS_ARGS[@]}" "$TRIPLE" + # FIXME: The following is a workaround for the issue with wrong libxml2.a path + local LIBXML2_PATH_TO_FIX="$TARGET_BUILD_ROOT/wasi-sysroot/$CLANG_MULTIARCH_TRIPLE/lib/libxml2.a" + if [[ -f "$LIBXML2_PATH_TO_FIX" ]]; then + cp "$LIBXML2_PATH_TO_FIX" "$TRIPLE_DESTDIR/usr/lib/swift_static/wasi/libxml2.a" + fi } main() { @@ -137,7 +111,8 @@ main() { # NOTE: The llvm-cmake-options is a workaround for the issue on amazonlinux2 # See https://github.com/apple/swift/commit/40c7268e8f7d402b27e3ad16a84180e07c37f92c - "$SOURCE_PATH/swift/utils/build-script" \ + # NOTE: Add llvm-bin directory to PATH so that wasmstdlib.py can find FileCheck during tests + env PATH="$OPTIONS_LLVM_BIN:$OPTIONS_SWIFT_BIN:$PATH" "$SOURCE_PATH/swift/utils/build-script" \ --build-subdir=WebAssembly \ --release \ --skip-build-llvm \ @@ -145,11 +120,13 @@ main() { --skip-build-cmark \ --skip-build-benchmarks \ --skip-early-swift-driver \ + --wasmkit \ --build-wasm-stdlib \ --skip-test-wasm-stdlib \ --native-swift-tools-path="$OPTIONS_SWIFT_BIN" \ --native-clang-tools-path="$OPTIONS_CLANG_BIN" \ --native-llvm-tools-path="$OPTIONS_LLVM_BIN" \ + --build-runtime-with-host-compiler \ --extra-cmake-options="\ -DSWIFT_STDLIB_TRACING=NO \ -DSWIFT_STDLIB_HAS_ASL=NO \ @@ -168,13 +145,10 @@ main() { "$OPTIONS_SWIFT_BIN" ) - build_target_toolchain "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasi" "wasi-wasm32" "wasm32-wasi" "wasmstdlib" "wasi" + build_target_toolchain "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasip1" "wasip1-wasm32" "wasm32-wasip1" "wasmstdlib" "wasip1" build_target_toolchain "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasip1-threads" "wasip1-threads-wasm32" "wasm32-wasip1-threads" "wasmthreadsstdlib" "wasip1" rsync -av "$WASI_SYSROOT_PATH/" "$PACKAGING_DIR/wasi-sysroot/" - - build_target_corelibs "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasi" "wasm32-wasi" - build_target_corelibs "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasip1-threads" "wasm32-wasip1-threads" } main "$@" diff --git a/schemes/main/build/build-xctest.sh b/schemes/main/build/build-xctest.sh deleted file mode 100755 index 472cc664..00000000 --- a/schemes/main/build/build-xctest.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -ex -DESTINATION_TOOLCHAIN=$1 -LLVM_BIN_DIR=$2 -CLANG_BIN_DIR=$3 -SWIFT_BIN_DIR=$4 -WASI_SYSROOT_PATH=$5 -TRIPLE="$6" - -SOURCE_PATH="$(cd "$(dirname $0)/../../../../" && pwd)" -SCHEME_BUILD_PATH="$(cd "$(dirname $0)" && pwd)" - -BUILD_DIR="$SOURCE_PATH/build/WebAssembly/xctest-$TRIPLE" - -mkdir -p $BUILD_DIR -cd $BUILD_DIR - -swift_extra_flags="" -if [[ "$TRIPLE" == "wasm32-unknown-wasip1-threads" ]]; then - swift_extra_flags="-Xcc -matomics -Xcc -mbulk-memory -Xcc -mthread-model -Xcc posix -Xcc -pthread -Xcc -ftls-model=local-exec" -fi - -cmake -G Ninja \ - -DCMAKE_BUILD_TYPE="Release" \ - -DCMAKE_SYSROOT="$WASI_SYSROOT_PATH" \ - -DCMAKE_STAGING_PREFIX="$DESTINATION_TOOLCHAIN/usr" \ - -DCMAKE_Swift_COMPILER="$SWIFT_BIN_DIR/swiftc" \ - -DCMAKE_TOOLCHAIN_FILE="$SCHEME_BUILD_PATH/toolchain-wasi.cmake" \ - -DTRIPLE="$TRIPLE" \ - -DLLVM_BIN="$LLVM_BIN_DIR" \ - -DCLANG_BIN="$CLANG_BIN_DIR" \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_Swift_COMPILER_FORCED=ON \ - -DCMAKE_Swift_FLAGS="-sdk $WASI_SYSROOT_PATH -resource-dir $DESTINATION_TOOLCHAIN/usr/lib/swift_static $swift_extra_flags" \ - -DSWIFT_FOUNDATION_PATH=$DESTINATION_TOOLCHAIN/usr/lib/swift_static/wasi/wasm32 \ - "${SOURCE_PATH}/swift-corelibs-xctest" - -ninja -v -ninja -v install diff --git a/schemes/main/build/run-test.sh b/schemes/main/build/run-test.sh index aa92d6f8..701e6c60 100755 --- a/schemes/main/build/run-test.sh +++ b/schemes/main/build/run-test.sh @@ -2,9 +2,4 @@ set -euxo pipefail -SOURCE_PATH="$(cd "$(dirname "$0")/../../../.." && pwd)" -BUILD_DIR="$SOURCE_PATH/build" -TARGET_BUILD_ROOT="$BUILD_DIR/WebAssembly" - -HOST_SUFFIX=$(find "$TARGET_BUILD_ROOT" -name "wasmstdlib-*" -exec basename {} \; | sed 's/wasmstdlib-//') -env "PATH=$TARGET_BUILD_ROOT/llvm-$HOST_SUFFIX/bin:$BUILD_DIR/llvm-tools/bin:$PATH" "LIT_FILTER_OUT=(IRGen/|embedded/)" ninja check-swift-wasi-wasm32-custom -C "$TARGET_BUILD_ROOT/wasmstdlib-$HOST_SUFFIX" +echo "Skipping test because we now run tests in build-script" diff --git a/schemes/main/manifest.json b/schemes/main/manifest.json index 47605521..8867f7f9 100644 --- a/schemes/main/manifest.json +++ b/schemes/main/manifest.json @@ -1,9 +1,6 @@ { - "base-tag": "swift-DEVELOPMENT-SNAPSHOT-2025-07-18-a", + "base-tag": "swift-DEVELOPMENT-SNAPSHOT-2025-07-23-a", "icu4c": [], - "libxml2": [ - "https://github.com/swiftwasm/libxml2-wasm/releases/download/2.0.0/libxml2-wasm32-unknown-wasi.tar.gz", - "https://github.com/swiftwasm/libxml2-wasm/releases/download/2.0.0/libxml2-wasm32-unknown-wasip1-threads.tar.gz" - ], + "libxml2": [], "swift-org-download-channel": "development" } diff --git a/test/swift-sdk/foundation/ProcessInfo.swift b/test/swift-sdk/foundation/ProcessInfo.swift index 74053dd1..1b1ef371 100644 --- a/test/swift-sdk/foundation/ProcessInfo.swift +++ b/test/swift-sdk/foundation/ProcessInfo.swift @@ -1,8 +1,6 @@ // RUN: %{target_simple_swift_build} // RUN: %{wasm_run} --dir %t.dir::/tmp %t.dir/.build/debug/Check.wasm | %{FileCheck} %s // REQUIRES: FileCheck && scheme=main -// XFAIL: scheme=main -// https://github.com/swiftlang/swift-foundation/pull/1447 import Foundation diff --git a/tools/build/build-toolchain b/tools/build/build-toolchain new file mode 100755 index 00000000..93a25182 --- /dev/null +++ b/tools/build/build-toolchain @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import sys +import argparse +import pathlib +import subprocess +from build_support.actions import derive_options_from_args, REPO_ROOT, DownloadBaseSnapshotAction + + +def main(): + parser = argparse.ArgumentParser(description='Build the Swift toolchain') + options = derive_options_from_args(sys.argv[1:], parser) + + # Calculate paths + repo_path = pathlib.Path(REPO_ROOT) + source_path = repo_path.parent + tools_build_path = pathlib.Path(__file__).parent + + build_dir = source_path / "build" + cross_compiler_destdir = pathlib.Path(DownloadBaseSnapshotAction.toolchain_path(options)) + + # Install base toolchain + print("=====> Installing base toolchain") + subprocess.check_call([ + str(tools_build_path / "install-base-toolchain"), + "--scheme", options.scheme + ]) + + # Build LLVM tools + print("=====> Building LLVM tools") + subprocess.check_call([ + str(tools_build_path / "build-llvm-tools"), + "--toolchain", str(cross_compiler_destdir) + ]) + + # Build target toolchain + print("=====> Building target toolchain") + scheme_build_script = repo_path / "schemes" / options.scheme / "build" / "build-target-toolchain.sh" + + subprocess.check_call([ + str(scheme_build_script), + "--llvm-bin", str(build_dir / "llvm-tools" / "bin"), + "--clang-bin", str(cross_compiler_destdir / "usr" / "bin"), + "--swift-bin", str(cross_compiler_destdir / "usr" / "bin"), + "--scheme", options.scheme + ]) + + +if __name__ == '__main__': + main() diff --git a/tools/build/build-toolchain.sh b/tools/build/build-toolchain.sh deleted file mode 100755 index 1823d9a3..00000000 --- a/tools/build/build-toolchain.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -eux -o pipefail - -REPO_PATH="$(cd "$(dirname $0)/../.." && pwd)" -SOURCE_PATH="$REPO_PATH/.." -SCHEME="${1:?"scheme is not specified"}" - -TOOLS_BUILD_PATH="$(cd "$(dirname "$0")" && pwd)" - -BUILD_DIR="$SOURCE_PATH/build" -PACKAGING_DIR="$BUILD_DIR/Packaging" -CROSS_COMPILER_DESTDIR=$PACKAGING_DIR/base-snapshot - -"$TOOLS_BUILD_PATH/install-base-toolchain" --scheme "$SCHEME" -"$TOOLS_BUILD_PATH/build-llvm-tools" --toolchain "$CROSS_COMPILER_DESTDIR" - -"$REPO_PATH/schemes/$SCHEME/build/build-target-toolchain.sh" \ - --llvm-bin "$BUILD_DIR/llvm-tools/bin" \ - --clang-bin "$CROSS_COMPILER_DESTDIR/usr/bin" \ - --swift-bin "$CROSS_COMPILER_DESTDIR/usr/bin" \ - --scheme "$SCHEME" - diff --git a/tools/build/build_support/actions.py b/tools/build/build_support/actions.py index fb16c6d1..126dce2b 100644 --- a/tools/build/build_support/actions.py +++ b/tools/build/build_support/actions.py @@ -114,16 +114,25 @@ def run(self): self.system('git', '-C', f'../{repo}', 'checkout', rev) class DownloadBaseSnapshotAction(Action): + + @staticmethod + def toolchains_dir(): + return os.path.join(REPO_ROOT, '..', 'build', 'Packaging', 'base-toolchain') + + @staticmethod + def toolchain_path(options): + return os.path.join(DownloadBaseSnapshotAction.toolchains_dir(), options.tag) + def run(self): platform_info = platform.PlatformInfo.derive() snapshot_url, tarball_name = platform_info.snapshot_url(self.options.swift_org_download_channel, self.options.tag) extension = platform_info.package_extension - tarball_path = os.path.join('..', 'build', 'Packaging', f'base-snapshot.{extension}') - base_snapshot_dir = os.path.join('..', 'build', 'Packaging', 'base-snapshot') + base_toolchain_dir = DownloadBaseSnapshotAction.toolchain_path(self.options) + tarball_path = os.path.join(os.path.dirname(base_toolchain_dir), tarball_name) - if os.path.exists(os.path.join(base_snapshot_dir, 'usr')): - print(f"=====> Base snapshot '{base_snapshot_dir}' already exists") + if os.path.exists(os.path.join(base_toolchain_dir, 'usr')): + print(f"=====> Base snapshot '{base_toolchain_dir}' already exists") return if not os.path.exists(tarball_path): @@ -132,15 +141,15 @@ def run(self): self.system('curl', '--fail', '-L', '-o', tarball_path, snapshot_url) print(f"=====> Unpacking base snapshot {tarball_name}") - os.makedirs(base_snapshot_dir, exist_ok=True) + os.makedirs(base_toolchain_dir, exist_ok=True) if extension == "tar.gz": - self.system('tar', '-C', base_snapshot_dir, '--strip-components', '1', '-xzf', tarball_path) + self.system('tar', '-C', base_toolchain_dir, '--strip-components', '1', '-xzf', tarball_path) elif extension == "pkg": import tempfile with tempfile.TemporaryDirectory() as tmpdir: self.system('xar', '-xf', tarball_path, '-C', tmpdir) old_cwd = os.getcwd() - os.chdir(base_snapshot_dir) + os.chdir(base_toolchain_dir) pkg_name = tarball_name.replace(".pkg", "-package.pkg") self.system('cpio', '-i', '-I', os.path.join(tmpdir, pkg_name, 'Payload')) os.chdir(old_cwd) diff --git a/tools/build/build_support/llvm_tools.py b/tools/build/build_support/llvm_tools.py index eb47da43..6e2f7e0e 100644 --- a/tools/build/build_support/llvm_tools.py +++ b/tools/build/build_support/llvm_tools.py @@ -8,4 +8,6 @@ # These tools are used by Swift test suite 'llvm-nm', 'FileCheck', -] \ No newline at end of file + 'split-file', + 'not', +] diff --git a/tools/build/ci.sh b/tools/build/ci.sh index f75c44d9..584e19d7 100755 --- a/tools/build/ci.sh +++ b/tools/build/ci.sh @@ -34,6 +34,6 @@ install_sccache_if_needed() { install_sccache_if_needed export PATH="$SCCACHE_INSTALL_PATH:$PATH" -$TOOLS_BUILD_PATH/build-toolchain.sh "$SCHEME" +$TOOLS_BUILD_PATH/build-toolchain --scheme "$SCHEME" $TOOLS_BUILD_PATH/package-toolchain --scheme "$SCHEME" --daily-snapshot diff --git a/tools/build/package-toolchain b/tools/build/package-toolchain index 9fe19402..4110c2aa 100755 --- a/tools/build/package-toolchain +++ b/tools/build/package-toolchain @@ -51,7 +51,10 @@ def copy_icu_libs(scheme, build_sdk_path, dist_toolchain_path, target_triple): shutil.copy(os.path.join(icu_lib_dir, lib), dest_path) -def copy_libxml2_libs(build_sdk_path, dist_toolchain_path, target_triple): +def copy_libxml2_libs(scheme, build_sdk_path, dist_toolchain_path, target_triple): + if scheme not in ["release-6.0", "release-6.1"]: + # Recent channels install libxml2 in wasi-sysroot by default + return import shutil lib_dir = os.path.join(build_sdk_path, f"libxml2-{target_triple}", 'lib') if not os.path.exists(lib_dir): @@ -83,7 +86,7 @@ class PackageSwiftSDKAction(Action): def run(self): build_sdk_path = os.path.join('..', 'build-sdk') copy_icu_libs(self.options.scheme, build_sdk_path, self.target_toolchain_path, self.target_triple) - copy_libxml2_libs(build_sdk_path, self.target_toolchain_path, self.target_triple) + copy_libxml2_libs(self.options.scheme, build_sdk_path, self.target_toolchain_path, self.target_triple) self.make_swift_sdk( self.target_toolchain_path, self.wasi_sysroot_path, @@ -207,8 +210,12 @@ def main(): toolchain_channel = derive_toolchain_channel(options.scheme) triples = [ - ["wasm32-unknown-wasi", "wasm32-wasi"], + ["wasm32-unknown-wasip1", "wasm32-wasip1"], ] + if options.scheme in ["release-6.0", "release-6.1"]: + triples = [ + ["wasm32-unknown-wasi", "wasm32-wasi"], + ] if options.scheme not in ["release-6.0"]: triples += [ ["wasm32-unknown-wasip1-threads", "wasm32-wasip1-threads"] diff --git a/tools/build/run-e2e-test b/tools/build/run-e2e-test new file mode 100755 index 00000000..374c48c6 --- /dev/null +++ b/tools/build/run-e2e-test @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +import sys +import argparse +import pathlib +import subprocess +from build_support.actions import derive_options_from_args, REPO_ROOT, DownloadBaseSnapshotAction + + +def default_swift_sdk(options, swift_sdks_path: pathlib.Path): + """Return the default target triple based on the scheme.""" + import json + for artifact_bundle in swift_sdks_path.iterdir(): + if not artifact_bundle.is_dir() or not artifact_bundle.name.endswith('.artifactbundle'): + # Skip if not ending with .artifactbundle + continue + info_json = artifact_bundle / 'info.json' + if not info_json.is_file(): + continue + try: + with info_json.open('r') as f: + info = json.load(f) + + sdk_ids = info.get('artifacts', {}).keys() + for sdk_id in sdk_ids: + if sdk_id.endswith("wasm32-unknown-wasip1") or sdk_id.endswith("wasm32-unknown-wasi"): + return sdk_id + except Exception as e: + print(f"Error reading {info_json}: {e}", file=sys.stderr) + continue + + +def main(): + parser = argparse.ArgumentParser( + description='Run e2e integration tests for Swift SDK') + parser.add_argument('--swift-sdk', default=None, + help='Swift SDK identifier (default: wasm32-unknown-wasip1)') + parser.add_argument('args', nargs=argparse.REMAINDER, + help='Extra arguments to pass to lit.py') + options = derive_options_from_args(sys.argv[1:], parser) + + extra_lit_args = options.args + if len(extra_lit_args) > 0 and extra_lit_args[0] == "--": + extra_lit_args = extra_lit_args[1:] + + repo_path = pathlib.Path(REPO_ROOT) + base_toolchain_path = DownloadBaseSnapshotAction.toolchain_path(options) + lit_py_path = repo_path.parent / "llvm-project" / "llvm" / "utils" / "lit" / "lit.py" + swift_sdks_path = repo_path.parent / "swift-sdk-generator" / "Bundles" + swift_sdk = options.swift_sdk or default_swift_sdk(options, swift_sdks_path) + + # Build lit command + lit_cmd = [ + str(lit_py_path), + "--param", f"swift-sdk={swift_sdk}", + "--param", f"scheme={options.scheme}", + "--param", f"base-toolchain-path={base_toolchain_path}", + "--param", f"swift-sdks-path={swift_sdks_path}", + ] + test_dir = repo_path / "test" + lit_cmd.append(str(test_dir)) + lit_cmd.extend(extra_lit_args) + + print("=====> Running e2e integration tests") + print(f"Command: {' '.join(lit_cmd)}") + + # Run the tests + try: + subprocess.check_call(lit_cmd) + print("=====> e2e tests completed successfully") + except subprocess.CalledProcessError as e: + print(f"=====> e2e tests failed with exit code {e.returncode}") + sys.exit(e.returncode) + + +if __name__ == '__main__': + main()