From bc95e4d165e770648dd049c6c55de039c2ec6195 Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Fri, 14 Feb 2025 08:40:17 +0000 Subject: [PATCH 01/10] wip: Try container-based SDK build in CI --- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index ffbfc26d..747e4f17 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -381,12 +381,10 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase { } func testAarch64FromContainer() async throws { - try skipSlow() try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) } func testX86_64FromContainer() async throws { - try skipSlow() try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) } } From 9a67e7dcd3f419a687a84a72c5c0973b1f77da0f Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Fri, 14 Feb 2025 08:45:28 +0000 Subject: [PATCH 02/10] remove CI check --- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 747e4f17..f4528509 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -115,9 +115,9 @@ final class RepeatedBuildTests: XCTestCase { private let logger = Logger(label: "swift-sdk-generator") func testRepeatedSDKBuilds() async throws { - if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { - throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") - } +// if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { +// throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") +// } var logger = logger logger[metadataKey: "testcase"] = "testRepeatedSDKBuilds" From abe93a8c92b241f7277eab5fac43bb8816835d38 Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Fri, 14 Feb 2025 08:46:30 +0000 Subject: [PATCH 03/10] build 5.9 SDKs to match earliest version which runs in CI --- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index f4528509..0982674e 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -323,12 +323,10 @@ final class Swift59_UbuntuEndToEndTests: XCTestCase { } func testAarch64FromContainer() async throws { - try skipSlow() try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) } func testX86_64FromContainer() async throws { - try skipSlow() try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) } } @@ -381,10 +379,12 @@ final class Swift60_UbuntuEndToEndTests: XCTestCase { } func testAarch64FromContainer() async throws { + try skipSlow() try await buildTestcases(config: config.withArchitecture("aarch64").withDocker()) } func testX86_64FromContainer() async throws { + try skipSlow() try await buildTestcases(config: config.withArchitecture("x86_64").withDocker()) } } From 9614f9bb06ec28623cfecf874516b2cc462f4a21 Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Fri, 14 Feb 2025 09:03:32 +0000 Subject: [PATCH 04/10] wip: try podman as runtime in CI --- Docker/Dockerfile | 2 +- Docker/docker-compose.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index ba606a49..36c31c4d 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -7,7 +7,7 @@ ARG swift_version ARG ubuntu_version # set as UTF-8 -RUN apt-get update && apt-get install -y locales locales-all libsqlite3-dev +RUN apt-get update && apt-get install -y locales locales-all libsqlite3-dev podman ENV LC_ALL en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 diff --git a/Docker/docker-compose.yaml b/Docker/docker-compose.yaml index 3db9408e..22c41acf 100644 --- a/Docker/docker-compose.yaml +++ b/Docker/docker-compose.yaml @@ -27,6 +27,7 @@ services: <<: *common environment: - JENKINS_URL + - SWIFT_SDK_GENERATOR_CONTAINER_RUNTIME=podman command: /bin/bash -xcl "swift test $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-}" # util From bb2bffa241e1890c892673d7c1dceccd9c7b1c98 Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Fri, 14 Feb 2025 09:10:49 +0000 Subject: [PATCH 05/10] wip: force tests to use podman as well --- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 0982674e..d19d184d 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -126,7 +126,7 @@ final class RepeatedBuildTests: XCTestCase { // Test with no arguments by default: var possibleArguments = ["--host-toolchain"] do { - try await Shell.run("docker ps") + try await Shell.run("podman ps") possibleArguments.append("--with-docker --linux-distribution-name rhel --linux-distribution-version ubi9") } catch { self.logger.warning("Docker CLI does not seem to be working, skipping tests that involve Docker.") @@ -231,7 +231,7 @@ func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempD logger.info("Building test project in 6.0-\(containerVersion) container") buildOutput = try await Shell.readStdout( """ - docker run --rm -v \(testPackageDir):/src \ + podman run --rm -v \(testPackageDir):/src \ -v $HOME/.swiftpm/swift-sdks:/root/.swiftpm/swift-sdks \ --workdir /src swift:6.0-\(containerVersion) \ /bin/bash -c "swift build --scratch-path /root/.build --experimental-swift-sdk \(bundleName)" @@ -243,7 +243,7 @@ func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempD logger.info("Building test project in 6.0-\(containerVersion) container with static-swift-stdlib") buildOutput = try await Shell.readStdout( """ - docker run --rm -v \(testPackageDir):/src \ + podman run --rm -v \(testPackageDir):/src \ -v $HOME/.swiftpm/swift-sdks:/root/.swiftpm/swift-sdks \ --workdir /src swift:6.0-\(containerVersion) \ /bin/bash -c "swift build --scratch-path /root/.build --experimental-swift-sdk \(bundleName) --static-swift-stdlib" @@ -281,7 +281,7 @@ func buildTestcases(config: SDKConfiguration) async throws { if config.withDocker { do { - try await Shell.run("docker ps") + try await Shell.run("podman ps") } catch { throw XCTSkip("Container runtime is not available - skipping tests which require it") } From ff9e63bc0e5989419d1247bd20f4423508d93c63 Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Fri, 14 Feb 2025 09:11:07 +0000 Subject: [PATCH 06/10] wip: Force no host toolchain to avoid downloads --- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index d19d184d..0dd28948 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -124,7 +124,7 @@ final class RepeatedBuildTests: XCTestCase { // Test that an existing SDK can be rebuilt without cleaning up. // Test with no arguments by default: - var possibleArguments = ["--host-toolchain"] + var possibleArguments = ["--no-host-toolchain"] do { try await Shell.run("podman ps") possibleArguments.append("--with-docker --linux-distribution-name rhel --linux-distribution-version ubi9") @@ -176,7 +176,7 @@ struct SDKConfiguration { var sdkGeneratorArguments: String { return [ "--sdk-name \(bundleName)", - "--host-toolchain", + "--no-host-toolchain", withDocker ? "--with-docker" : nil, "--swift-version \(swiftVersion)-RELEASE", testLinuxSwiftSDKs ? "--host \(hostArch!)-unknown-linux-gnu" : nil, From 70ea6a3536855cbe2302a0fbee3c3fcbfaad092f Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Fri, 14 Feb 2025 09:36:03 +0000 Subject: [PATCH 07/10] wip: run tests in a privileged container --- Docker/docker-compose.2204.59.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Docker/docker-compose.2204.59.yaml b/Docker/docker-compose.2204.59.yaml index a335b481..f33fc281 100644 --- a/Docker/docker-compose.2204.59.yaml +++ b/Docker/docker-compose.2204.59.yaml @@ -10,6 +10,7 @@ services: test: image: swift-sdk-generator:22.04-5.9 + privileged: true environment: # - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors # need to fix S-CL-F sendability first - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error From e8bdb219fd652d37caa3b7170f2948e9c63c7dbe Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Fri, 14 Feb 2025 09:47:46 +0000 Subject: [PATCH 08/10] wip: move privileged to common compose file --- Docker/docker-compose.2204.59.yaml | 1 - Docker/docker-compose.yaml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/docker-compose.2204.59.yaml b/Docker/docker-compose.2204.59.yaml index f33fc281..a335b481 100644 --- a/Docker/docker-compose.2204.59.yaml +++ b/Docker/docker-compose.2204.59.yaml @@ -10,7 +10,6 @@ services: test: image: swift-sdk-generator:22.04-5.9 - privileged: true environment: # - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors # need to fix S-CL-F sendability first - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error diff --git a/Docker/docker-compose.yaml b/Docker/docker-compose.yaml index 22c41acf..67ad1755 100644 --- a/Docker/docker-compose.yaml +++ b/Docker/docker-compose.yaml @@ -25,6 +25,7 @@ services: test: <<: *common + privileged: true environment: - JENKINS_URL - SWIFT_SDK_GENERATOR_CONTAINER_RUNTIME=podman From 0a34063c9bc757dfefa7ca02e6a6de94caa3e29d Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Fri, 14 Feb 2025 09:51:35 +0000 Subject: [PATCH 09/10] wip: only run a representative end to end test to save time --- Docker/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/docker-compose.yaml b/Docker/docker-compose.yaml index 67ad1755..0c8c0da0 100644 --- a/Docker/docker-compose.yaml +++ b/Docker/docker-compose.yaml @@ -29,7 +29,7 @@ services: environment: - JENKINS_URL - SWIFT_SDK_GENERATOR_CONTAINER_RUNTIME=podman - command: /bin/bash -xcl "swift test $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-}" + command: /bin/bash -xcl "swift test $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} --filter Swift59_UbuntuEndToEndTests.testX86_64FromContainer" # util From 9496975be8eebf9bbb1a5f48b8d4a487f605c10c Mon Sep 17 00:00:00 2001 From: Euan Harris Date: Fri, 14 Feb 2025 10:47:55 +0000 Subject: [PATCH 10/10] wip: allow end to end tests to run in CI --- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 0dd28948..8fde90e5 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -275,9 +275,9 @@ func buildTestcases(config: SDKConfiguration) async throws { var logger = Logger(label: "EndToEndTests") logger[metadataKey: "testcase"] = "testPackageInitExecutable" - if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { - throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") - } +// if ProcessInfo.processInfo.environment.keys.contains("JENKINS_URL") { +// throw XCTSkip("EndToEnd tests cannot currently run in CI: https://github.com/swiftlang/swift-sdk-generator/issues/145") +// } if config.withDocker { do {