From 492e84c149db586773780ccfb1bd378aeb6a0b53 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Mon, 15 Jul 2024 16:19:25 -0400 Subject: [PATCH 1/4] Detect when Swift Testing returns `EXIT_NO_TESTS_FOUND`. (#7777) This PR detects when a Swift Testing run returns `EXIT_NO_TESTS_FOUND` and treats it as a successful test run. This change handles the change from https://github.com/apple/swift-testing/pull/536. Separately, the refactor in #7766 stops Swift Package Manager from reporting `"No matching test cases were run"` when XCTest has no tests matching passed `--filter` arguments. A third PR after these two have been merged will restore that functionality by collating results from both XCTest and Swift Testing. > [!NOTE] > Test coverage can be added after Swift Testing is in the Swift toolchain. This change partially resolves rdar://131704587. --- Sources/Commands/SwiftTestCommand.swift | 71 +++++++++++++++++++++---- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index 8ff59d6d2f0..c59b7815296 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -37,6 +37,10 @@ import var TSCBasic.stdoutStream import class TSCBasic.SynchronizedQueue import class TSCBasic.Thread +#if os(Windows) +import WinSDK // for ERROR_NOT_FOUND +#endif + private enum TestError: Swift.Error { case invalidListTestJSONData(context: String, underlyingError: Error? = nil) case testsNotFound @@ -862,13 +866,40 @@ final class TestRunner { /// Executes and returns execution status. Prints test output on standard streams if requested /// - Returns: Boolean indicating if test execution returned code 0, and the output stream result - public func test(outputHandler: @escaping (String) -> Void) -> Bool { - var success = true + func test(outputHandler: @escaping (String) -> Void) -> Bool { + (test(outputHandler: outputHandler) as Result) != .failure + } + + /// The result of running the test(s). + enum Result: Equatable { + /// The test(s) ran successfully. + case success + + /// The test(s) failed. + case failure + + /// There were no matching tests to run. + /// + /// XCTest does not report this result. It is used by Swift Testing only. + case noMatchingTests + } + + /// Executes and returns execution status. Prints test output on standard streams if requested + /// - Returns: Result of spawning and running the test process, and the output stream result + @_disfavoredOverload + func test(outputHandler: @escaping (String) -> Void) -> Result { + var results = [Result]() for path in self.bundlePaths { let testSuccess = self.test(at: path, outputHandler: outputHandler) - success = success && testSuccess + results.append(testSuccess) + } + if results.contains(.failure) { + return .failure + } else if results.isEmpty || results.contains(.success) { + return .success + } else { + return .noMatchingTests } - return success } /// Constructs arguments to execute XCTest. @@ -892,7 +923,7 @@ final class TestRunner { return args } - private func test(at path: AbsolutePath, outputHandler: @escaping (String) -> Void) -> Bool { + private func test(at path: AbsolutePath, outputHandler: @escaping (String) -> Void) -> Result { let testObservabilityScope = self.observabilityScope.makeChildScope(description: "running test at \(path)") do { @@ -907,25 +938,27 @@ final class TestRunner { ) let process = AsyncProcess(arguments: try args(forTestAt: path), environment: self.testEnv, outputRedirection: outputRedirection) guard let terminationKey = self.cancellator.register(process) else { - return false // terminating + return .failure // terminating } defer { self.cancellator.deregister(terminationKey) } try process.launch() let result = try process.waitUntilExit() switch result.exitStatus { case .terminated(code: 0): - return true + return .success + case .terminated(code: EXIT_NO_TESTS_FOUND) where library == .swiftTesting: + return .noMatchingTests #if !os(Windows) case .signalled(let signal) where ![SIGINT, SIGKILL, SIGTERM].contains(signal): testObservabilityScope.emit(error: "Exited with unexpected signal code \(signal)") - return false + return .failure #endif default: - return false + return .failure } } catch { testObservabilityScope.emit(error) - return false + return .failure } } } @@ -1392,6 +1425,24 @@ private extension Basics.Diagnostic { } } +/// The exit code returned to Swift Package Manager by Swift Testing when no +/// tests matched the inputs specified by the developer (or, for the case of +/// `swift test list`, when no tests were found.) +/// +/// Because Swift Package Manager does not directly link to the testing library, +/// it duplicates the definition of this constant in its own source. Any changes +/// to this constant in either package must be mirrored in the other. +private var EXIT_NO_TESTS_FOUND: CInt { +#if os(macOS) || os(Linux) + EX_UNAVAILABLE +#elseif os(Windows) + ERROR_NOT_FOUND +#else +#warning("Platform-specific implementation missing: value for EXIT_NO_TESTS_FOUND unavailable") + return 2 // We're assuming that EXIT_SUCCESS = 0 and EXIT_FAILURE = 1. +#endif +} + /// Builds the "test" target if enabled in options. /// /// - Returns: The paths to the build test products. From 2921bd9c8aab20c35dd6983d16a142e3eac0af98 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 16 Jul 2024 15:23:33 -0400 Subject: [PATCH 2/4] Build one test product for both Swift Testing and XCTest (with Swift Testing enabled by default.) (#7766) This PR refactors the previously-experimental Swift Testing support logic so that only a single build product is produced when using both XCTest and Swift Testing, and detection of Swift Testing usage is no longer needed at compile time. On macOS, Xcode 16 is responsible for hosting Swift Testing content, so additional changes may be needed in Xcode to support this refactoring. Such changes are beyond the purview of the Swift open source project. Resolves rdar://120864035. --------- Co-authored-by: Pavel Yaskevich --- .../LLBuildManifestBuilder.swift | 8 +- .../Build/BuildPlan/BuildPlan+Product.swift | 6 +- Sources/Build/BuildPlan/BuildPlan+Test.swift | 26 +- Sources/Build/BuildPlan/BuildPlan.swift | 42 +- Sources/Build/LLBuildCommands.swift | 252 +++++----- Sources/Build/LLBuildDescription.swift | 3 +- Sources/Commands/PackageCommands/Init.swift | 11 +- Sources/Commands/SwiftBuildCommand.swift | 25 +- Sources/Commands/SwiftTestCommand.swift | 439 +++++++++--------- .../Commands/Utilities/PluginDelegate.swift | 6 +- .../Commands/Utilities/TestingSupport.swift | 32 +- Sources/CoreCommands/Options.swift | 93 +--- Sources/PackageModel/UserToolchain.swift | 18 + .../BuildParameters+Testing.swift | 20 +- .../BuildParameters/BuildParameters.swift | 15 +- Sources/SPMBuildCore/BuiltTestProduct.swift | 17 +- Sources/Workspace/InitPackage.swift | 2 +- Sources/XCBuildSupport/XcodeBuildSystem.swift | 3 +- Tests/BuildTests/BuildPlanTests.swift | 21 +- Tests/CommandsTests/TestCommandTests.swift | 1 - Tests/WorkspaceTests/InitTests.swift | 6 +- 21 files changed, 471 insertions(+), 575 deletions(-) diff --git a/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift b/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift index a8adc3923e6..7a0c293105d 100644 --- a/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift +++ b/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift @@ -112,9 +112,7 @@ public class LLBuildManifestBuilder { } } - if self.plan.destinationBuildParameters.testingParameters.library == .xctest { - try self.addTestDiscoveryGenerationCommand() - } + try self.addTestDiscoveryGenerationCommand() try self.addTestEntryPointGenerationCommand() // Create command for all products in the plan. @@ -310,9 +308,7 @@ extension LLBuildManifestBuilder { let outputs = testEntryPointTarget.target.sources.paths - let mainFileName = TestEntryPointTool.mainFileName( - for: self.plan.destinationBuildParameters.testingParameters.library - ) + let mainFileName = TestEntryPointTool.mainFileName guard let mainOutput = (outputs.first { $0.basename == mainFileName }) else { throw InternalError("main output (\(mainFileName)) not found") } diff --git a/Sources/Build/BuildPlan/BuildPlan+Product.swift b/Sources/Build/BuildPlan/BuildPlan+Product.swift index 3967aef58f8..9782103e689 100644 --- a/Sources/Build/BuildPlan/BuildPlan+Product.swift +++ b/Sources/Build/BuildPlan/BuildPlan+Product.swift @@ -275,10 +275,8 @@ extension BuildPlan { } // Add derived test targets, if necessary - if buildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets { - if product.type == .test, let derivedTestTargets = derivedTestTargetsMap[product.id] { - staticTargets.append(contentsOf: derivedTestTargets) - } + if product.type == .test, let derivedTestTargets = derivedTestTargetsMap[product.id] { + staticTargets.append(contentsOf: derivedTestTargets) } return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths, providedLibraries, availableTools) diff --git a/Sources/Build/BuildPlan/BuildPlan+Test.swift b/Sources/Build/BuildPlan/BuildPlan+Test.swift index 53cce82bc97..96b20532d8e 100644 --- a/Sources/Build/BuildPlan/BuildPlan+Test.swift +++ b/Sources/Build/BuildPlan/BuildPlan+Test.swift @@ -34,13 +34,12 @@ extension BuildPlan { _ fileSystem: FileSystem, _ observabilityScope: ObservabilityScope ) throws -> [(product: ResolvedProduct, discoveryTargetBuildDescription: SwiftModuleBuildDescription?, entryPointTargetBuildDescription: SwiftModuleBuildDescription)] { - guard destinationBuildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets, - case .entryPointExecutable(let explicitlyEnabledDiscovery, let explicitlySpecifiedPath) = - destinationBuildParameters.testingParameters.testProductStyle - else { - throw InternalError("makeTestManifestTargets should not be used for build plan which does not require additional derived test targets") + var explicitlyEnabledDiscovery = false + var explicitlySpecifiedPath: AbsolutePath? + if case let .entryPointExecutable(caseExplicitlyEnabledDiscovery, caseExplicitlySpecifiedPath) = destinationBuildParameters.testingParameters.testProductStyle { + explicitlyEnabledDiscovery = caseExplicitlyEnabledDiscovery + explicitlySpecifiedPath = caseExplicitlySpecifiedPath } - let isEntryPointPathSpecifiedExplicitly = explicitlySpecifiedPath != nil var isDiscoveryEnabledRedundantly = explicitlyEnabledDiscovery && !isEntryPointPathSpecifiedExplicitly @@ -116,7 +115,7 @@ extension BuildPlan { resolvedTargetDependencies: [ResolvedModule.Dependency] ) throws -> SwiftModuleBuildDescription { let entryPointDerivedDir = destinationBuildParameters.buildPath.appending(components: "\(testProduct.name).derived") - let entryPointMainFileName = TestEntryPointTool.mainFileName(for: destinationBuildParameters.testingParameters.library) + let entryPointMainFileName = TestEntryPointTool.mainFileName let entryPointMainFile = entryPointDerivedDir.appending(component: entryPointMainFileName) let entryPointSources = Sources(paths: [entryPointMainFile], root: entryPointDerivedDir) @@ -153,18 +152,17 @@ extension BuildPlan { let swiftTargetDependencies: [Module.Dependency] let resolvedTargetDependencies: [ResolvedModule.Dependency] - switch destinationBuildParameters.testingParameters.library { - case .xctest: + if destinationBuildParameters.triple.isDarwin() { + discoveryTargets = nil + swiftTargetDependencies = [] + resolvedTargetDependencies = [] + } else { discoveryTargets = try generateDiscoveryTargets() swiftTargetDependencies = [.module(discoveryTargets!.target, conditions: [])] resolvedTargetDependencies = [.module(discoveryTargets!.resolved, conditions: [])] - case .swiftTesting: - discoveryTargets = nil - swiftTargetDependencies = testProduct.modules.map { .module($0.underlying, conditions: []) } - resolvedTargetDependencies = testProduct.modules.map { .module($0, conditions: []) } } - if let entryPointResolvedTarget = testProduct.testEntryPointModule { + if !destinationBuildParameters.triple.isDarwin(), let entryPointResolvedTarget = testProduct.testEntryPointModule { if isEntryPointPathSpecifiedExplicitly || explicitlyEnabledDiscovery { if isEntryPointPathSpecifiedExplicitly { // Allow using the explicitly-specified test entry point target, but still perform test discovery and thus declare a dependency on the discovery modules. diff --git a/Sources/Build/BuildPlan/BuildPlan.swift b/Sources/Build/BuildPlan/BuildPlan.swift index 12504f0dbb1..e2ce60f9a48 100644 --- a/Sources/Build/BuildPlan/BuildPlan.swift +++ b/Sources/Build/BuildPlan/BuildPlan.swift @@ -443,31 +443,29 @@ public class BuildPlan: SPMBuildCore.BuildPlan { } // Plan the derived test targets, if necessary. - if destinationBuildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets { - let derivedTestTargets = try Self.makeDerivedTestTargets( - testProducts: productMap.values.filter { - $0.product.type == .test - }, - destinationBuildParameters: destinationBuildParameters, - toolsBuildParameters: toolsBuildParameters, - shouldDisableSandbox: self.shouldDisableSandbox, - self.fileSystem, - self.observabilityScope - ) - for item in derivedTestTargets { - var derivedTestTargets = [item.entryPointTargetBuildDescription.target] - - targetMap[item.entryPointTargetBuildDescription.target.id] = .swift( - item.entryPointTargetBuildDescription - ) + let derivedTestTargets = try Self.makeDerivedTestTargets( + testProducts: productMap.values.filter { + $0.product.type == .test + }, + destinationBuildParameters: destinationBuildParameters, + toolsBuildParameters: toolsBuildParameters, + shouldDisableSandbox: self.shouldDisableSandbox, + self.fileSystem, + self.observabilityScope + ) + for item in derivedTestTargets { + var derivedTestTargets = [item.entryPointTargetBuildDescription.target] - if let discoveryTargetBuildDescription = item.discoveryTargetBuildDescription { - targetMap[discoveryTargetBuildDescription.target.id] = .swift(discoveryTargetBuildDescription) - derivedTestTargets.append(discoveryTargetBuildDescription.target) - } + targetMap[item.entryPointTargetBuildDescription.target.id] = .swift( + item.entryPointTargetBuildDescription + ) - self.derivedTestTargetsMap[item.product.id] = derivedTestTargets + if let discoveryTargetBuildDescription = item.discoveryTargetBuildDescription { + targetMap[discoveryTargetBuildDescription.target.id] = .swift(discoveryTargetBuildDescription) + derivedTestTargets.append(discoveryTargetBuildDescription.target) } + + self.derivedTestTargetsMap[item.product.id] = derivedTestTargets } self.productMap = productMap.mapValues(\.buildDescription) diff --git a/Sources/Build/LLBuildCommands.swift b/Sources/Build/LLBuildCommands.swift index a414119c236..f083947608b 100644 --- a/Sources/Build/LLBuildCommands.swift +++ b/Sources/Build/LLBuildCommands.swift @@ -50,8 +50,8 @@ extension IndexStore.TestCaseClass.TestMethod { } extension TestEntryPointTool { - public static func mainFileName(for library: BuildParameters.Testing.Library) -> String { - "runner-\(library).swift" + public static var mainFileName: String { + "runner.swift" } } @@ -105,74 +105,76 @@ final class TestDiscoveryCommand: CustomLLBuildCommand, TestBuildCommand { private func execute(fileSystem: Basics.FileSystem, tool: TestDiscoveryTool) throws { let outputs = tool.outputs.compactMap { try? AbsolutePath(validating: $0.name) } - switch self.context.productsBuildParameters.testingParameters.library { - case .swiftTesting: + if case .loadableBundle = context.productsBuildParameters.testingParameters.testProductStyle { + // When building an XCTest bundle, test discovery is handled by the + // test harness process (i.e. this is the Darwin path.) for file in outputs { try fileSystem.writeIfChanged(path: file, string: "") } - case .xctest: - let index = self.context.productsBuildParameters.indexStore - let api = try self.context.indexStoreAPI.get() - let store = try IndexStore.open(store: TSCAbsolutePath(index), api: api) - - // FIXME: We can speed this up by having one llbuild command per object file. - let tests = try store - .listTests(in: tool.inputs.map { try TSCAbsolutePath(AbsolutePath(validating: $0.name)) }) - - let testsByModule = Dictionary(grouping: tests, by: { $0.module.spm_mangledToC99ExtendedIdentifier() }) - - // Find the main file path. - guard let mainFile = outputs.first(where: { path in - path.basename == TestDiscoveryTool.mainFileName - }) else { - throw InternalError("main output (\(TestDiscoveryTool.mainFileName)) not found") - } + return + } - // Write one file for each test module. - // - // We could write everything in one file but that can easily run into type conflicts due - // in complex packages with large number of test modules. - for file in outputs where file != mainFile { - // FIXME: This is relying on implementation detail of the output but passing the - // the context all the way through is not worth it right now. - let module = file.basenameWithoutExt.spm_mangledToC99ExtendedIdentifier() - - guard let tests = testsByModule[module] else { - // This module has no tests so just write an empty file for it. - try fileSystem.writeFileContents(file, bytes: "") - continue - } - try write( - tests: tests, - forModule: module, - fileSystem: fileSystem, - path: file - ) + let index = self.context.productsBuildParameters.indexStore + let api = try self.context.indexStoreAPI.get() + let store = try IndexStore.open(store: TSCAbsolutePath(index), api: api) + + // FIXME: We can speed this up by having one llbuild command per object file. + let tests = try store + .listTests(in: tool.inputs.map { try TSCAbsolutePath(AbsolutePath(validating: $0.name)) }) + + let testsByModule = Dictionary(grouping: tests, by: { $0.module.spm_mangledToC99ExtendedIdentifier() }) + + // Find the main file path. + guard let mainFile = outputs.first(where: { path in + path.basename == TestDiscoveryTool.mainFileName + }) else { + throw InternalError("main output (\(TestDiscoveryTool.mainFileName)) not found") + } + + // Write one file for each test module. + // + // We could write everything in one file but that can easily run into type conflicts due + // in complex packages with large number of test modules. + for file in outputs where file != mainFile { + // FIXME: This is relying on implementation detail of the output but passing the + // the context all the way through is not worth it right now. + let module = file.basenameWithoutExt.spm_mangledToC99ExtendedIdentifier() + + guard let tests = testsByModule[module] else { + // This module has no tests so just write an empty file for it. + try fileSystem.writeFileContents(file, bytes: "") + continue } + try write( + tests: tests, + forModule: module, + fileSystem: fileSystem, + path: file + ) + } - let testsKeyword = tests.isEmpty ? "let" : "var" + let testsKeyword = tests.isEmpty ? "let" : "var" - // Write the main file. - let stream = try LocalFileOutputByteStream(mainFile) + // Write the main file. + let stream = try LocalFileOutputByteStream(mainFile) - stream.send( - #""" - import XCTest + stream.send( + #""" + import XCTest - @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") - @MainActor - public func __allDiscoveredTests() -> [XCTestCaseEntry] { - \#(testsKeyword) tests = [XCTestCaseEntry]() + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") + @MainActor + public func __allDiscoveredTests() -> [XCTestCaseEntry] { + \#(testsKeyword) tests = [XCTestCaseEntry]() - \#(testsByModule.keys.map { "tests += __\($0)__allTests()" }.joined(separator: "\n ")) + \#(testsByModule.keys.map { "tests += __\($0)__allTests()" }.joined(separator: "\n ")) - return tests - } - """# - ) + return tests + } + """# + ) - stream.flush() - } + stream.flush() } override func execute( @@ -201,9 +203,7 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { let outputs = tool.outputs.compactMap { try? AbsolutePath(validating: $0.name) } // Find the main output file - let mainFileName = TestEntryPointTool.mainFileName( - for: self.context.productsBuildParameters.testingParameters.library - ) + let mainFileName = TestEntryPointTool.mainFileName guard let mainFile = outputs.first(where: { path in path.basename == mainFileName }) else { @@ -213,62 +213,100 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { // Write the main file. let stream = try LocalFileOutputByteStream(mainFile) - switch self.context.productsBuildParameters.testingParameters.library { - case .swiftTesting: - stream.send( - #""" - #if canImport(Testing) - import Testing - #endif + // Find the inputs, which are the names of the test discovery module(s) + let inputs = tool.inputs.compactMap { try? AbsolutePath(validating: $0.name) } + let discoveryModuleNames = inputs.map(\.basenameWithoutExt) - @main struct Runner { - static func main() async { - #if canImport(Testing) - await Testing.__swiftPMEntryPoint() as Never - #endif + let testObservabilitySetup: String + let buildParameters = self.context.productsBuildParameters + if buildParameters.testingParameters.experimentalTestOutput && buildParameters.triple.supportsTestSummary { + testObservabilitySetup = "_ = SwiftPMXCTestObserver()\n" + } else { + testObservabilitySetup = "" + } + + let isXCTMainAvailable: String = switch buildParameters.testingParameters.testProductStyle { + case .entryPointExecutable: + "canImport(XCTest)" + case .loadableBundle: + "false" + } + + /// On WASI, we can't block the main thread, so XCTestMain is defined as async. + let awaitXCTMainKeyword = if buildParameters.triple.isWASI() { + "await" + } else { + "" + } + + var needsAsyncMainWorkaround = false + if buildParameters.triple.isLinux() { + // FIXME: work around crash on Amazon Linux 2 when main function is async (rdar://128303921) + needsAsyncMainWorkaround = true + } else if buildParameters.triple.isDarwin() { +#if compiler(<5.10) + // FIXME: work around duplicate async_Main symbols (SEE https://github.com/swiftlang/swift/pull/69113) + needsAsyncMainWorkaround = true +#endif + } + + stream.send( + #""" + #if canImport(Testing) + import Testing + #endif + + #if \#(isXCTMainAvailable) + \#(generateTestObservationCode(buildParameters: buildParameters)) + + import XCTest + \#(discoveryModuleNames.map { "import \($0)" }.joined(separator: "\n")) + #endif + + @main + @available(macOS 10.15, iOS 11, watchOS 4, tvOS 11, *) + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") + struct Runner { + private static func testingLibrary() -> String { + var iterator = CommandLine.arguments.makeIterator() + while let argument = iterator.next() { + if argument == "--testing-library", let libraryName = iterator.next() { + return libraryName.lowercased() + } } - } - """# - ) - case .xctest: - // Find the inputs, which are the names of the test discovery module(s) - let inputs = tool.inputs.compactMap { try? AbsolutePath(validating: $0.name) } - let discoveryModuleNames = inputs.map(\.basenameWithoutExt) - - let testObservabilitySetup: String - let buildParameters = self.context.productsBuildParameters - if buildParameters.testingParameters.experimentalTestOutput && buildParameters.triple.supportsTestSummary { - testObservabilitySetup = "_ = SwiftPMXCTestObserver()\n" - } else { - testObservabilitySetup = "" - } - stream.send( - #""" - \#(generateTestObservationCode(buildParameters: buildParameters)) + // Fallback if not specified: run XCTest (legacy behavior) + return "xctest" + } - import XCTest - \#(discoveryModuleNames.map { "import \($0)" }.joined(separator: "\n")) + #if \#(needsAsyncMainWorkaround) + @_silgen_name("$ss13_runAsyncMainyyyyYaKcF") + private static func _runAsyncMain(_ asyncFun: @Sendable @escaping () async throws -> ()) + #endif - @main - @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") - struct Runner { - #if os(WASI) - /// On WASI, we can't block the main thread, so XCTestMain is defined as async. - static func main() async { - \#(testObservabilitySetup) - await XCTMain(__allDiscoveredTests()) as Never + static func main() \#(needsAsyncMainWorkaround ? "" : "async") { + let testingLibrary = Self.testingLibrary() + #if canImport(Testing) + if testingLibrary == "swift-testing" { + #if \#(needsAsyncMainWorkaround) + _runAsyncMain { + await Testing.__swiftPMEntryPoint() as Never + } + #else + await Testing.__swiftPMEntryPoint() as Never + #endif } - #else - static func main() { + #endif + #if \#(isXCTMainAvailable) + if testingLibrary == "xctest" { \#(testObservabilitySetup) - XCTMain(__allDiscoveredTests()) as Never + \#(awaitXCTMainKeyword) XCTMain(__allDiscoveredTests()) as Never } #endif } - """# - ) - } + } + """# + ) stream.flush() } diff --git a/Sources/Build/LLBuildDescription.swift b/Sources/Build/LLBuildDescription.swift index 5612d33918e..bafa740ba1c 100644 --- a/Sources/Build/LLBuildDescription.swift +++ b/Sources/Build/LLBuildDescription.swift @@ -111,8 +111,7 @@ public struct BuildDescription: Codable { try BuiltTestProduct( productName: desc.product.name, binaryPath: desc.binaryPath, - packagePath: desc.package.path, - library: desc.buildParameters.testingParameters.library + packagePath: desc.package.path ) } self.pluginDescriptions = pluginDescriptions diff --git a/Sources/Commands/PackageCommands/Init.swift b/Sources/Commands/PackageCommands/Init.swift index 6263e84b835..8fd9522df7e 100644 --- a/Sources/Commands/PackageCommands/Init.swift +++ b/Sources/Commands/PackageCommands/Init.swift @@ -54,20 +54,11 @@ extension SwiftPackageCommand { throw InternalError("Could not find the current working directory") } - // NOTE: Do not use testLibraryOptions.enabledTestingLibraries(swiftCommandState:) here - // because the package doesn't exist yet, so there are no dependencies for it to query. - var testingLibraries: Set = [] - if testLibraryOptions.enableXCTestSupport { - testingLibraries.insert(.xctest) - } - if testLibraryOptions.explicitlyEnableSwiftTestingLibrarySupport == true { - testingLibraries.insert(.swiftTesting) - } let packageName = self.packageName ?? cwd.basename let initPackage = try InitPackage( name: packageName, packageType: initMode, - supportedTestingLibraries: testingLibraries, + supportedTestingLibraries: Set(testLibraryOptions.enabledTestingLibraries), destinationPath: cwd, installedSwiftPMConfiguration: swiftCommandState.getHostToolchain().installedSwiftPMConfiguration, fileSystem: swiftCommandState.fileSystem diff --git a/Sources/Commands/SwiftBuildCommand.swift b/Sources/Commands/SwiftBuildCommand.swift index 53c8db111a1..1a0ce6b23a0 100644 --- a/Sources/Commands/SwiftBuildCommand.swift +++ b/Sources/Commands/SwiftBuildCommand.swift @@ -159,35 +159,12 @@ public struct SwiftBuildCommand: AsyncSwiftCommand { var productsBuildParameters = try swiftCommandState.productsBuildParameters var toolsBuildParameters = try swiftCommandState.toolsBuildParameters - // Clean out the code coverage directory that may contain stale - // profraw files from a previous run of the code coverage tool. if self.options.enableCodeCoverage { - try swiftCommandState.fileSystem.removeFileTree(swiftCommandState.productsBuildParameters.codeCovPath) productsBuildParameters.testingParameters.enableCodeCoverage = true toolsBuildParameters.testingParameters.enableCodeCoverage = true } - if case .allIncludingTests = subset { - func updateTestingParameters(of buildParameters: inout BuildParameters, library: BuildParameters.Testing.Library) { - buildParameters.testingParameters = .init( - configuration: buildParameters.configuration, - targetTriple: buildParameters.triple, - enableCodeCoverage: buildParameters.testingParameters.enableCodeCoverage, - enableTestability: buildParameters.testingParameters.enableTestability, - experimentalTestOutput: buildParameters.testingParameters.experimentalTestOutput, - forceTestDiscovery: globalOptions.build.enableTestDiscovery, - testEntryPointPath: globalOptions.build.testEntryPointPath, - library: library - ) - } - for library in try options.testLibraryOptions.enabledTestingLibraries(swiftCommandState: swiftCommandState) { - updateTestingParameters(of: &productsBuildParameters, library: library) - updateTestingParameters(of: &toolsBuildParameters, library: library) - try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) - } - } else { - try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) - } + try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) } private func build( diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index c59b7815296..2902b006de1 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -245,102 +245,116 @@ public struct SwiftTestCommand: AsyncSwiftCommand { @OptionGroup() var options: TestCommandOptions - // MARK: - XCTest - - private func xctestRun(_ swiftCommandState: SwiftCommandState) async throws { - // validate XCTest available on darwin based systems - let toolchain = try swiftCommandState.getTargetToolchain() - if case let .unsupported(reason) = try swiftCommandState.getHostToolchain().swiftSDK.xctestSupport { - if let reason { - throw TestError.xctestNotAvailable(reason: reason) - } else { - throw TestError.xcodeNotInstalled - } - } else if toolchain.targetTriple.isDarwin() && toolchain.xctestPath == nil { - throw TestError.xcodeNotInstalled - } - - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: .xctest) - + private func run(_ swiftCommandState: SwiftCommandState, buildParameters: BuildParameters, testProducts: [BuiltTestProduct]) async throws { // Remove test output from prior runs and validate priors. - if self.options.enableExperimentalTestOutput && productsBuildParameters.triple.supportsTestSummary { - _ = try? localFileSystem.removeFileTree(productsBuildParameters.testOutputPath) + if self.options.enableExperimentalTestOutput && buildParameters.triple.supportsTestSummary { + _ = try? localFileSystem.removeFileTree(buildParameters.testOutputPath) } - let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState, library: .xctest) - if !self.options.shouldRunInParallel { - let xctestArgs = try xctestArgs(for: testProducts, swiftCommandState: swiftCommandState) - try await runTestProducts( - testProducts, - additionalArguments: xctestArgs, - productsBuildParameters: productsBuildParameters, - swiftCommandState: swiftCommandState, - library: .xctest - ) - } else { - let testSuites = try TestingSupport.getTestSuites( - in: testProducts, - swiftCommandState: swiftCommandState, - enableCodeCoverage: options.enableCodeCoverage, - shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, - experimentalTestOutput: options.enableExperimentalTestOutput, - sanitizers: globalOptions.build.sanitizers - ) - let tests = try testSuites - .filteredTests(specifier: options.testCaseSpecifier) - .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) - - // If there were no matches, emit a warning and exit. - if tests.isEmpty { - swiftCommandState.observabilityScope.emit(.noMatchingTests) - try generateXUnitOutputIfRequested(for: [], swiftCommandState: swiftCommandState) - return - } + var results = [TestRunner.Result]() - // Clean out the code coverage directory that may contain stale - // profraw files from a previous run of the code coverage tool. - if self.options.enableCodeCoverage { - try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) + // Run XCTest. + if options.testLibraryOptions.isEnabled(.xctest) { + // validate XCTest available on darwin based systems + let toolchain = try swiftCommandState.getTargetToolchain() + if case let .unsupported(reason) = try swiftCommandState.getHostToolchain().swiftSDK.xctestSupport { + if let reason { + throw TestError.xctestNotAvailable(reason: reason) + } else { + throw TestError.xcodeNotInstalled + } + } else if toolchain.targetTriple.isDarwin() && toolchain.xctestPath == nil { + throw TestError.xcodeNotInstalled } - // Run the tests using the parallel runner. - let runner = ParallelTestRunner( - bundlePaths: testProducts.map { $0.bundlePath }, - cancellator: swiftCommandState.cancellator, - toolchain: toolchain, - numJobs: options.numberOfWorkers ?? ProcessInfo.processInfo.activeProcessorCount, - buildOptions: globalOptions.build, - productsBuildParameters: productsBuildParameters, - shouldOutputSuccess: swiftCommandState.logLevel <= .info, - observabilityScope: swiftCommandState.observabilityScope - ) - - let testResults = try runner.run(tests) + if !self.options.shouldRunInParallel { + let (xctestArgs, testCount) = try xctestArgs(for: testProducts, swiftCommandState: swiftCommandState) + let result = try await runTestProducts( + testProducts, + additionalArguments: xctestArgs, + productsBuildParameters: buildParameters, + swiftCommandState: swiftCommandState, + library: .xctest + ) + if result == .success && testCount == 0 { + results.append(.noMatchingTests) + } else { + results.append(result) + } + } else { + let testSuites = try TestingSupport.getTestSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + enableCodeCoverage: options.enableCodeCoverage, + shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, + experimentalTestOutput: options.enableExperimentalTestOutput, + sanitizers: globalOptions.build.sanitizers + ) + let tests = try testSuites + .filteredTests(specifier: options.testCaseSpecifier) + .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) + + let result: TestRunner.Result + let testResults: [ParallelTestRunner.TestResult] + if tests.isEmpty { + testResults = [] + result = .noMatchingTests + } else { + // Run the tests using the parallel runner. + let runner = ParallelTestRunner( + bundlePaths: testProducts.map { $0.bundlePath }, + cancellator: swiftCommandState.cancellator, + toolchain: toolchain, + numJobs: options.numberOfWorkers ?? ProcessInfo.processInfo.activeProcessorCount, + buildOptions: globalOptions.build, + productsBuildParameters: buildParameters, + shouldOutputSuccess: swiftCommandState.logLevel <= .info, + observabilityScope: swiftCommandState.observabilityScope + ) - try generateXUnitOutputIfRequested(for: testResults, swiftCommandState: swiftCommandState) + testResults = try runner.run(tests) + result = runner.ranSuccessfully ? .success : .failure + } - // Process code coverage if requested - if self.options.enableCodeCoverage, runner.ranSuccessfully { - try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState, library: .xctest) + try generateXUnitOutputIfRequested(for: testResults, swiftCommandState: swiftCommandState) + results.append(result) } + } - if !runner.ranSuccessfully { - swiftCommandState.executionStatus = .failure - } + // Run Swift Testing (parallel or not, it has a single entry point.) + if options.testLibraryOptions.isEnabled(.swiftTesting) { + results.append( + try await runTestProducts( + testProducts, + additionalArguments: [], + productsBuildParameters: buildParameters, + swiftCommandState: swiftCommandState, + library: .swiftTesting + ) + ) + } - if self.options.enableExperimentalTestOutput, !runner.ranSuccessfully { - try Self.handleTestOutput(productsBuildParameters: productsBuildParameters, packagePath: testProducts[0].packagePath) + switch results.reduce() { + case .success: + // Nothing to do here. + break + case .failure: + swiftCommandState.executionStatus = .failure + if self.options.enableExperimentalTestOutput { + try Self.handleTestOutput(productsBuildParameters: buildParameters, packagePath: testProducts[0].packagePath) } + case .noMatchingTests: + swiftCommandState.observabilityScope.emit(.noMatchingTests) } } - private func xctestArgs(for testProducts: [BuiltTestProduct], swiftCommandState: SwiftCommandState) throws -> [String] { + private func xctestArgs(for testProducts: [BuiltTestProduct], swiftCommandState: SwiftCommandState) throws -> (arguments: [String], testCount: Int) { switch options.testCaseSpecifier { case .none: if case .skip = options.skippedTests(fileSystem: swiftCommandState.fileSystem) { fallthrough } else { - return [] + return ([], 0) } case .regex, .specific, .skip: @@ -362,12 +376,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { .filteredTests(specifier: options.testCaseSpecifier) .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) - // If there were no matches, emit a warning. - if tests.isEmpty { - swiftCommandState.observabilityScope.emit(.noMatchingTests) - } - - return TestRunner.xctestArguments(forTestSpecifiers: tests.map(\.specifier)) + return (TestRunner.xctestArguments(forTestSpecifiers: tests.map(\.specifier)), tests.count) } } @@ -387,21 +396,6 @@ public struct SwiftTestCommand: AsyncSwiftCommand { try generator.generate(at: xUnitOutput) } - // MARK: - swift-testing - - private func swiftTestingRun(_ swiftCommandState: SwiftCommandState) async throws { - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: .swiftTesting) - let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState, library: .swiftTesting) - let additionalArguments = Array(CommandLine.arguments.dropFirst()) - try await runTestProducts( - testProducts, - additionalArguments: additionalArguments, - productsBuildParameters: productsBuildParameters, - swiftCommandState: swiftCommandState, - library: .swiftTesting - ) - } - // MARK: - Common implementation public func run(_ swiftCommandState: SwiftCommandState) async throws { @@ -420,12 +414,22 @@ public struct SwiftTestCommand: AsyncSwiftCommand { let command = try List.parse() try command.run(swiftCommandState) } else { - if try options.testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { - try await swiftTestingRun(swiftCommandState) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) + let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState) + + // Clean out the code coverage directory that may contain stale + // profraw files from a previous run of the code coverage tool. + if self.options.enableCodeCoverage { + try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) } - if options.testLibraryOptions.enableXCTestSupport { - try await xctestRun(swiftCommandState) + + try await run(swiftCommandState, buildParameters: productsBuildParameters, testProducts: testProducts) + + // process code Coverage if request + if self.options.enableCodeCoverage, swiftCommandState.executionStatus != .failure { + try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState) } + } } @@ -435,11 +439,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { productsBuildParameters: BuildParameters, swiftCommandState: SwiftCommandState, library: BuildParameters.Testing.Library - ) async throws { - // Clean out the code coverage directory that may contain stale - // profraw files from a previous run of the code coverage tool. - if self.options.enableCodeCoverage { - try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) + ) async throws -> TestRunner.Result { + // Pass through all arguments from the command line to Swift Testing. + var additionalArguments = additionalArguments + if library == .swiftTesting { + additionalArguments += CommandLine.arguments.dropFirst() } let toolchain = try swiftCommandState.getTargetToolchain() @@ -450,8 +454,15 @@ public struct SwiftTestCommand: AsyncSwiftCommand { library: library ) + let runnerPaths: [AbsolutePath] = switch library { + case .xctest: + testProducts.map(\.bundlePath) + case .swiftTesting: + testProducts.map(\.binaryPath) + } + let runner = TestRunner( - bundlePaths: testProducts.map { library == .xctest ? $0.bundlePath : $0.binaryPath }, + bundlePaths: runnerPaths, additionalArguments: additionalArguments, cancellator: swiftCommandState.cancellator, toolchain: toolchain, @@ -461,22 +472,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { ) // Finally, run the tests. - let ranSuccessfully = runner.test(outputHandler: { + return runner.test(outputHandler: { // command's result output goes on stdout // ie "swift test" should output to stdout print($0, terminator: "") }) - if !ranSuccessfully { - swiftCommandState.executionStatus = .failure - } - - if self.options.enableCodeCoverage, ranSuccessfully { - try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState, library: library) - } - - if self.options.enableExperimentalTestOutput, !ranSuccessfully { - try Self.handleTestOutput(productsBuildParameters: productsBuildParameters, packagePath: testProducts[0].packagePath) - } } private static func handleTestOutput(productsBuildParameters: BuildParameters, packagePath: AbsolutePath) throws { @@ -513,8 +513,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { /// Processes the code coverage data and emits a json. private func processCodeCoverage( _ testProducts: [BuiltTestProduct], - swiftCommandState: SwiftCommandState, - library: BuildParameters.Testing.Library + swiftCommandState: SwiftCommandState ) async throws { let workspace = try swiftCommandState.getActiveWorkspace() let root = try swiftCommandState.getWorkspaceRoot() @@ -527,23 +526,23 @@ public struct SwiftTestCommand: AsyncSwiftCommand { } // Merge all the profraw files to produce a single profdata file. - try mergeCodeCovRawDataFiles(swiftCommandState: swiftCommandState, library: library) + try mergeCodeCovRawDataFiles(swiftCommandState: swiftCommandState) - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) for product in testProducts { // Export the codecov data as JSON. let jsonPath = productsBuildParameters.codeCovAsJSONPath(packageName: rootManifest.displayName) - try exportCodeCovAsJSON(to: jsonPath, testBinary: product.binaryPath, swiftCommandState: swiftCommandState, library: library) + try exportCodeCovAsJSON(to: jsonPath, testBinary: product.binaryPath, swiftCommandState: swiftCommandState) } } /// Merges all profraw profiles in codecoverage directory into default.profdata file. - private func mergeCodeCovRawDataFiles(swiftCommandState: SwiftCommandState, library: BuildParameters.Testing.Library) throws { + private func mergeCodeCovRawDataFiles(swiftCommandState: SwiftCommandState) throws { // Get the llvm-prof tool. let llvmProf = try swiftCommandState.getTargetToolchain().getLLVMProf() // Get the profraw files. - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) let codeCovFiles = try swiftCommandState.fileSystem.getDirectoryContents(productsBuildParameters.codeCovPath) // Construct arguments for invoking the llvm-prof tool. @@ -563,12 +562,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { private func exportCodeCovAsJSON( to path: AbsolutePath, testBinary: AbsolutePath, - swiftCommandState: SwiftCommandState, - library: BuildParameters.Testing.Library + swiftCommandState: SwiftCommandState ) throws { // Export using the llvm-cov tool. let llvmCov = try swiftCommandState.getTargetToolchain().getLLVMCov() - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) let args = [ llvmCov.pathString, "export", @@ -588,10 +586,9 @@ public struct SwiftTestCommand: AsyncSwiftCommand { /// /// - Returns: The paths to the build test products. private func buildTestsIfNeeded( - swiftCommandState: SwiftCommandState, - library: BuildParameters.Testing.Library + swiftCommandState: SwiftCommandState ) throws -> [BuiltTestProduct] { - let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest(options: self.options) return try Commands.buildTestsIfNeeded( swiftCommandState: swiftCommandState, productsBuildParameters: productsBuildParameters, @@ -618,7 +615,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { throw StringError("'--num-workers' must be greater than zero") } - if !options.testLibraryOptions.enableXCTestSupport { + guard options.testLibraryOptions.isEnabled(.xctest) else { throw StringError("'--num-workers' is only supported when testing with XCTest") } } @@ -645,7 +642,7 @@ extension SwiftTestCommand { guard let rootManifest = rootManifests.values.first else { throw StringError("invalid manifests at \(root.packages)") } - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(enableCodeCoverage: true, library: .xctest) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(enableCodeCoverage: true) print(productsBuildParameters.codeCovAsJSONPath(packageName: rootManifest.displayName)) } } @@ -686,41 +683,10 @@ extension SwiftTestCommand { @Flag(name: [.customLong("list-tests"), .customShort("l")], help: .hidden) var _deprecated_passthrough: Bool = false - // MARK: - XCTest - - private func xctestRun(_ swiftCommandState: SwiftCommandState) throws { - let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest( - enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding, - library: .xctest - ) - let testProducts = try buildTestsIfNeeded( - swiftCommandState: swiftCommandState, - productsBuildParameters: productsBuildParameters, - toolsBuildParameters: toolsBuildParameters - ) - let testSuites = try TestingSupport.getTestSuites( - in: testProducts, - swiftCommandState: swiftCommandState, - enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding, - experimentalTestOutput: false, - sanitizers: globalOptions.build.sanitizers - ) - - // Print the tests. - for test in testSuites.allTests { - print(test.specifier) - } - } - - // MARK: - swift-testing - - private func swiftTestingRun(_ swiftCommandState: SwiftCommandState) throws { + func run(_ swiftCommandState: SwiftCommandState) throws { let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest( enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding, - library: .swiftTesting + shouldSkipBuilding: sharedOptions.shouldSkipBuilding ) let testProducts = try buildTestsIfNeeded( swiftCommandState: swiftCommandState, @@ -736,36 +702,43 @@ extension SwiftTestCommand { library: .swiftTesting ) - let additionalArguments = ["--list-tests"] + CommandLine.arguments.dropFirst() - let runner = TestRunner( - bundlePaths: testProducts.map(\.binaryPath), - additionalArguments: additionalArguments, - cancellator: swiftCommandState.cancellator, - toolchain: toolchain, - testEnv: testEnv, - observabilityScope: swiftCommandState.observabilityScope, - library: .swiftTesting - ) - - // Finally, run the tests. - let ranSuccessfully = runner.test(outputHandler: { - // command's result output goes on stdout - // ie "swift test" should output to stdout - print($0, terminator: "") - }) - if !ranSuccessfully { - swiftCommandState.executionStatus = .failure + if testLibraryOptions.isEnabled(.xctest) { + let testSuites = try TestingSupport.getTestSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + enableCodeCoverage: false, + shouldSkipBuilding: sharedOptions.shouldSkipBuilding, + experimentalTestOutput: false, + sanitizers: globalOptions.build.sanitizers + ) + + // Print the tests. + for test in testSuites.allTests { + print(test.specifier) + } } - } - - // MARK: - Common implementation - func run(_ swiftCommandState: SwiftCommandState) throws { - if try testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { - try swiftTestingRun(swiftCommandState) - } - if testLibraryOptions.enableXCTestSupport { - try xctestRun(swiftCommandState) + if testLibraryOptions.isEnabled(.swiftTesting) { + let additionalArguments = ["--list-tests"] + CommandLine.arguments.dropFirst() + let runner = TestRunner( + bundlePaths: testProducts.map(\.binaryPath), + additionalArguments: additionalArguments, + cancellator: swiftCommandState.cancellator, + toolchain: toolchain, + testEnv: testEnv, + observabilityScope: swiftCommandState.observabilityScope, + library: .swiftTesting + ) + + // Finally, run the tests. + let result = runner.test(outputHandler: { + // command's result output goes on stdout + // ie "swift test" should output to stdout + print($0, terminator: "") + }) + if result == .failure { + swiftCommandState.executionStatus = .failure + } } } @@ -864,12 +837,6 @@ final class TestRunner { self.library = library } - /// Executes and returns execution status. Prints test output on standard streams if requested - /// - Returns: Boolean indicating if test execution returned code 0, and the output stream result - func test(outputHandler: @escaping (String) -> Void) -> Bool { - (test(outputHandler: outputHandler) as Result) != .failure - } - /// The result of running the test(s). enum Result: Equatable { /// The test(s) ran successfully. @@ -886,39 +853,41 @@ final class TestRunner { /// Executes and returns execution status. Prints test output on standard streams if requested /// - Returns: Result of spawning and running the test process, and the output stream result - @_disfavoredOverload func test(outputHandler: @escaping (String) -> Void) -> Result { var results = [Result]() for path in self.bundlePaths { let testSuccess = self.test(at: path, outputHandler: outputHandler) results.append(testSuccess) } - if results.contains(.failure) { - return .failure - } else if results.isEmpty || results.contains(.success) { - return .success - } else { - return .noMatchingTests - } + return results.reduce() } /// Constructs arguments to execute XCTest. private func args(forTestAt testPath: AbsolutePath) throws -> [String] { var args: [String] = [] - #if os(macOS) - if library == .xctest { +#if os(macOS) + switch library { + case .xctest: guard let xctestPath = self.toolchain.xctestPath else { throw TestError.xcodeNotInstalled } - args = [xctestPath.pathString] - args += additionalArguments - args += [testPath.pathString] - return args + args += [xctestPath.pathString] + case .swiftTesting: + let helper = try self.toolchain.getSwiftTestingHelper() + args += [helper.pathString, "--test-bundle-path", testPath.pathString] } - #endif - - args += [testPath.description] args += additionalArguments + args += [testPath.pathString] +#else + args += [testPath.pathString] + args += additionalArguments +#endif + + if library == .swiftTesting { + // HACK: tell the test bundle/executable that we want to run Swift Testing, not XCTest. + // XCTest doesn't understand this argument (yet), so don't pass it there. + args += ["--testing-library", "swift-testing"] + } return args } @@ -963,6 +932,19 @@ final class TestRunner { } } +extension Collection where Element == TestRunner.Result { + /// Reduce all results in this collection into a single result. + func reduce() -> Element { + if contains(.failure) { + return .failure + } else if isEmpty || contains(.success) { + return .success + } else { + return .noMatchingTests + } + } +} + /// A class to run tests in parallel. final class ParallelTestRunner { /// An enum representing result of a unit test execution. @@ -1094,20 +1076,20 @@ final class ParallelTestRunner { toolchain: self.toolchain, testEnv: testEnv, observabilityScope: self.observabilityScope, - library: .xctest + library: .xctest // swift-testing does not use ParallelTestRunner ) var output = "" let outputLock = NSLock() let start = DispatchTime.now() - let success = testRunner.test(outputHandler: { _output in outputLock.withLock{ output += _output }}) + let result = testRunner.test(outputHandler: { _output in outputLock.withLock{ output += _output }}) let duration = start.distance(to: .now()) - if !success { + if result == .failure { self.ranSuccessfully = false } self.finishedTests.enqueue(TestResult( unitTest: test, output: output, - success: success, + success: result != .failure, duration: duration )) } @@ -1358,21 +1340,14 @@ final class XUnitGenerator { extension SwiftCommandState { func buildParametersForTest( - options: TestCommandOptions, - library: BuildParameters.Testing.Library + options: TestCommandOptions ) throws -> (productsBuildParameters: BuildParameters, toolsBuildParameters: BuildParameters) { - var result = try self.buildParametersForTest( + try self.buildParametersForTest( enableCodeCoverage: options.enableCodeCoverage, enableTestability: options.enableTestableImports, shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, - experimentalTestOutput: options.enableExperimentalTestOutput, - library: library + experimentalTestOutput: options.enableExperimentalTestOutput ) - if try options.testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: self) { - result.productsBuildParameters.flags.swiftCompilerFlags += ["-DSWIFT_PM_SUPPORTS_SWIFT_TESTING"] - result.toolsBuildParameters.flags.swiftCompilerFlags += ["-DSWIFT_PM_SUPPORTS_SWIFT_TESTING"] - } - return result } } diff --git a/Sources/Commands/Utilities/PluginDelegate.swift b/Sources/Commands/Utilities/PluginDelegate.swift index a118930abec..94c6bf3fffe 100644 --- a/Sources/Commands/Utilities/PluginDelegate.swift +++ b/Sources/Commands/Utilities/PluginDelegate.swift @@ -284,11 +284,11 @@ final class PluginDelegate: PluginInvocationDelegate { // Run the test — for now we run the sequentially so we can capture accurate timing results. let startTime = DispatchTime.now() - let success = testRunner.test(outputHandler: { _ in }) // this drops the tests output + let result = testRunner.test(outputHandler: { _ in }) // this drops the tests output let duration = Double(startTime.distance(to: .now()).milliseconds() ?? 0) / 1000.0 - numFailedTests += success ? 0 : 1 + numFailedTests += (result != .failure) ? 0 : 1 testResults.append( - .init(name: testName, result: success ? .succeeded : .failed, duration: duration) + .init(name: testName, result: (result != .failure) ? .succeeded : .failed, duration: duration) ) } diff --git a/Sources/Commands/Utilities/TestingSupport.swift b/Sources/Commands/Utilities/TestingSupport.swift index 59c7246e0a3..1291ac73e43 100644 --- a/Sources/Commands/Utilities/TestingSupport.swift +++ b/Sources/Commands/Utilities/TestingSupport.swift @@ -118,8 +118,7 @@ enum TestingSupport { buildParameters: swiftCommandState.buildParametersForTest( enableCodeCoverage: enableCodeCoverage, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput, - library: .xctest + experimentalTestOutput: experimentalTestOutput ).productsBuildParameters, sanitizers: sanitizers, library: .xctest @@ -134,8 +133,7 @@ enum TestingSupport { toolchain: try swiftCommandState.getTargetToolchain(), buildParameters: swiftCommandState.buildParametersForTest( enableCodeCoverage: enableCodeCoverage, - shouldSkipBuilding: shouldSkipBuilding, - library: .xctest + shouldSkipBuilding: shouldSkipBuilding ).productsBuildParameters, sanitizers: sanitizers, library: .xctest @@ -164,10 +162,6 @@ enum TestingSupport { env["NO_COLOR"] = "1" } - // Set an environment variable to indicate which library's test product - // is being executed. - env["SWIFT_PM_TEST_LIBRARY"] = String(describing: library) - // Add the code coverage related variables. if buildParameters.testingParameters.enableCodeCoverage { // Defines the path at which the profraw files will be written on test execution. @@ -177,7 +171,7 @@ enum TestingSupport { // execution but is required when the tests are running in parallel as // SwiftPM repeatedly invokes the test binary with the test case name as // the filter. - let codecovProfile = buildParameters.buildPath.appending(components: "codecov", "default%m.profraw") + let codecovProfile = buildParameters.buildPath.appending(components: "codecov", "\(library)%m.profraw") env["LLVM_PROFILE_FILE"] = codecovProfile.pathString } #if !os(macOS) @@ -195,6 +189,11 @@ enum TestingSupport { env.appendPath(key: "DYLD_LIBRARY_PATH", value: sdkPlatformFrameworksPath.lib.pathString) } + // We aren't using XCTest's harness logic to run Swift Testing tests. + if library == .xctest { + env["SWIFT_TESTING_ENABLED"] = "0" + } + // Fast path when no sanitizers are enabled. if sanitizers.isEmpty { return env @@ -221,24 +220,21 @@ extension SwiftCommandState { enableCodeCoverage: Bool, enableTestability: Bool? = nil, shouldSkipBuilding: Bool = false, - experimentalTestOutput: Bool = false, - library: BuildParameters.Testing.Library + experimentalTestOutput: Bool = false ) throws -> (productsBuildParameters: BuildParameters, toolsBuildParameters: BuildParameters) { let productsBuildParameters = buildParametersForTest( modifying: try productsBuildParameters, enableCodeCoverage: enableCodeCoverage, enableTestability: enableTestability, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput, - library: library + experimentalTestOutput: experimentalTestOutput ) let toolsBuildParameters = buildParametersForTest( modifying: try toolsBuildParameters, enableCodeCoverage: enableCodeCoverage, enableTestability: enableTestability, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput, - library: library + experimentalTestOutput: experimentalTestOutput ) return (productsBuildParameters, toolsBuildParameters) } @@ -248,8 +244,7 @@ extension SwiftCommandState { enableCodeCoverage: Bool, enableTestability: Bool?, shouldSkipBuilding: Bool, - experimentalTestOutput: Bool, - library: BuildParameters.Testing.Library + experimentalTestOutput: Bool ) -> BuildParameters { var parameters = parameters @@ -266,8 +261,7 @@ extension SwiftCommandState { configuration: parameters.configuration, targetTriple: parameters.triple, forceTestDiscovery: explicitlyEnabledDiscovery, - testEntryPointPath: explicitlySpecifiedPath, - library: library + testEntryPointPath: explicitlySpecifiedPath ) parameters.testingParameters.enableCodeCoverage = enableCodeCoverage diff --git a/Sources/CoreCommands/Options.swift b/Sources/CoreCommands/Options.swift index 20546383c09..eb6dd14a868 100644 --- a/Sources/CoreCommands/Options.swift +++ b/Sources/CoreCommands/Options.swift @@ -579,83 +579,36 @@ public struct TestLibraryOptions: ParsableArguments { help: "Enable support for XCTest") public var explicitlyEnableXCTestSupport: Bool? - /// Whether to enable support for XCTest. - public var enableXCTestSupport: Bool { - // Default to enabled. - explicitlyEnableXCTestSupport ?? true - } - - /// Whether to enable support for swift-testing (as explicitly specified by the user.) + /// Whether to enable support for Swift Testing (as explicitly specified by the user.) /// - /// Callers (other than `swift package init`) will generally want to use - /// ``enableSwiftTestingLibrarySupport(swiftCommandState:)`` since it will - /// take into account whether the package has a dependency on swift-testing. - @Flag(name: .customLong("experimental-swift-testing"), + /// Callers will generally want to use ``enableSwiftTestingLibrarySupport`` since it will + /// have the correct default value if the user didn't specify one. + @Flag(name: .customLong("swift-testing"), inversion: .prefixedEnableDisable, - help: "Enable experimental support for swift-testing") + help: "Enable support for Swift Testing") public var explicitlyEnableSwiftTestingLibrarySupport: Bool? - /// Whether to enable support for swift-testing. - public func enableSwiftTestingLibrarySupport( - swiftCommandState: SwiftCommandState - ) throws -> Bool { - // Honor the user's explicit command-line selection, if any. - if let callerSuppliedValue = explicitlyEnableSwiftTestingLibrarySupport { - return callerSuppliedValue - } - - // If the active package has a dependency on swift-testing, automatically enable support for it so that extra steps are not needed. - let workspace = try swiftCommandState.getActiveWorkspace() - let root = try swiftCommandState.getWorkspaceRoot() - let rootManifests = try temp_await { - workspace.loadRootManifests( - packages: root.packages, - observabilityScope: swiftCommandState.observabilityScope, - completion: $0 - ) - } - - // Is swift-testing among the dependencies of the package being built? - // If so, enable support. - let isEnabledByDependency = rootManifests.values.lazy - .flatMap(\.dependencies) - .map(\.identity) - .map(String.init(describing:)) - .contains("swift-testing") - if isEnabledByDependency { - swiftCommandState.observabilityScope.emit(debug: "Enabling swift-testing support due to its presence as a package dependency.") - return true - } - - // Is swift-testing the package being built itself (unlikely)? If so, - // enable support. - let isEnabledByName = root.packages.lazy - .map(PackageIdentity.init(path:)) - .map(String.init(describing:)) - .contains("swift-testing") - if isEnabledByName { - swiftCommandState.observabilityScope.emit(debug: "Enabling swift-testing support because it is a root package.") - return true + /// Legacy experimental equivalent of ``explicitlyEnableSwiftTestingLibrarySupport``. + /// + /// This option will be removed in a future update. + @Flag(name: .customLong("experimental-swift-testing"), + inversion: .prefixedEnableDisable, + help: .private) + public var explicitlyEnableExperimentalSwiftTestingLibrarySupport: Bool? + + /// Test whether or not a given library is enabled. + public func isEnabled(_ library: BuildParameters.Testing.Library) -> Bool { + switch library { + case .xctest: + explicitlyEnableXCTestSupport ?? true + case .swiftTesting: + explicitlyEnableSwiftTestingLibrarySupport ?? explicitlyEnableExperimentalSwiftTestingLibrarySupport ?? true } - - // Default to disabled since swift-testing is experimental (opt-in.) - return false } - /// Get the set of enabled testing libraries. - public func enabledTestingLibraries( - swiftCommandState: SwiftCommandState - ) throws -> Set { - var result = Set() - - if enableXCTestSupport { - result.insert(.xctest) - } - if try enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { - result.insert(.swiftTesting) - } - - return result + /// The list of enabled testing libraries. + public var enabledTestingLibraries: [BuildParameters.Testing.Library] { + [.xctest, .swiftTesting].filter(isEnabled) } } diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index 351321eecd8..2540f4384fa 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -387,6 +387,24 @@ public final class UserToolchain: Toolchain { ) } +#if os(macOS) + public func getSwiftTestingHelper() throws -> AbsolutePath { + // The helper would be located in `.build/` directory when + // SwiftPM is built locally and `usr/libexec/swift/pm` directory in + // an installed version. + let binDirectories = self.swiftSDK.toolset.rootPaths + + self.swiftSDK.toolset.rootPaths.map { + $0.parentDirectory.appending(components: ["libexec", "swift", "pm"]) + } + + return try UserToolchain.getTool( + "swiftpm-testing-helper", + binDirectories: binDirectories, + fileSystem: self.fileSystem + ) + } +#endif + internal static func deriveSwiftCFlags( triple: Triple, swiftSDK: SwiftSDK, diff --git a/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift b/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift index bdfb66bb6b8..697a24b65ff 100644 --- a/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift +++ b/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift @@ -38,17 +38,6 @@ extension BuildParameters { explicitlySpecifiedPath: AbsolutePath? ) - /// Whether this test product style requires additional, derived test targets, i.e. there must be additional test targets, beyond those - /// listed explicitly in the package manifest, created in order to add additional behavior (such as entry point logic). - public var requiresAdditionalDerivedTestTargets: Bool { - switch self { - case .loadableBundle: - return false - case .entryPointExecutable: - return true - } - } - /// The explicitly-specified entry point file path, if this style of test product supports it and a path was specified. public var explicitlySpecifiedEntryPointPath: AbsolutePath? { switch self { @@ -113,9 +102,6 @@ extension BuildParameters { } } - /// Which testing library to use for this build. - public var library: Library - public init( configuration: BuildConfiguration, targetTriple: Triple, @@ -123,8 +109,7 @@ extension BuildParameters { enableTestability: Bool? = nil, experimentalTestOutput: Bool = false, forceTestDiscovery: Bool = false, - testEntryPointPath: AbsolutePath? = nil, - library: Library = .xctest + testEntryPointPath: AbsolutePath? = nil ) { self.enableCodeCoverage = enableCodeCoverage self.experimentalTestOutput = experimentalTestOutput @@ -136,11 +121,10 @@ extension BuildParameters { // when building and testing in release mode, one can use the '--disable-testable-imports' flag // to disable testability in `swift test`, but that requires that the tests do not use the testable imports feature self.enableTestability = enableTestability ?? (.debug == configuration) - self.testProductStyle = (targetTriple.isDarwin() && library == .xctest) ? .loadableBundle : .entryPointExecutable( + self.testProductStyle = targetTriple.isDarwin() ? .loadableBundle : .entryPointExecutable( explicitlyEnabledDiscovery: forceTestDiscovery, explicitlySpecifiedPath: testEntryPointPath ) - self.library = library } } } diff --git a/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift b/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift index 5db1b5d6f9f..abade1ddf7d 100644 --- a/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift +++ b/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift @@ -293,16 +293,11 @@ public struct BuildParameters: Encodable { guard !self.triple.isWasm else { return try RelativePath(validating: "\(product.name).wasm") } - switch testingParameters.library { - case .xctest: - let base = "\(product.name).xctest" - if self.triple.isDarwin() { - return try RelativePath(validating: "\(base)/Contents/MacOS/\(product.name)") - } else { - return try RelativePath(validating: base) - } - case .swiftTesting: - return try RelativePath(validating: "\(product.name).swift-testing") + let base = "\(product.name).xctest" + if self.triple.isDarwin() { + return try RelativePath(validating: "\(base)/Contents/MacOS/\(product.name)") + } else { + return try RelativePath(validating: base) } case .macro: #if BUILD_MACROS_AS_DYLIBS diff --git a/Sources/SPMBuildCore/BuiltTestProduct.swift b/Sources/SPMBuildCore/BuiltTestProduct.swift index 881ade7175f..70f31901e3b 100644 --- a/Sources/SPMBuildCore/BuiltTestProduct.swift +++ b/Sources/SPMBuildCore/BuiltTestProduct.swift @@ -28,15 +28,8 @@ public struct BuiltTestProduct: Codable { /// When the test product is not bundled (for instance, when using XCTest on /// non-Darwin targets), this path is equal to ``binaryPath``. public var bundlePath: AbsolutePath { - // Go up the folder hierarchy until we find the .xctest or - // .swift-testing bundle. - let pathExtension: String - switch library { - case .xctest: - pathExtension = ".xctest" - case .swiftTesting: - pathExtension = ".swift-testing" - } + // Go up the folder hierarchy until we find the .xctest bundle. + let pathExtension = ".xctest" let hierarchySequence = sequence(first: binaryPath, next: { $0.isRoot ? nil : $0.parentDirectory }) guard let bundlePath = hierarchySequence.first(where: { $0.basename.hasSuffix(pathExtension) }) else { fatalError("could not find test bundle path from '\(binaryPath)'") @@ -45,18 +38,14 @@ public struct BuiltTestProduct: Codable { return bundlePath } - /// The library used to build this test product. - public var library: BuildParameters.Testing.Library - /// Creates a new instance. /// - Parameters: /// - productName: The test product name. /// - binaryPath: The path of the test binary. /// - packagePath: The path to the package this product was declared in. - public init(productName: String, binaryPath: AbsolutePath, packagePath: AbsolutePath, library: BuildParameters.Testing.Library) { + public init(productName: String, binaryPath: AbsolutePath, packagePath: AbsolutePath) { self.productName = productName self.binaryPath = binaryPath self.packagePath = packagePath - self.library = library } } diff --git a/Sources/Workspace/InitPackage.swift b/Sources/Workspace/InitPackage.swift index 78e3d783be2..866fb292835 100644 --- a/Sources/Workspace/InitPackage.swift +++ b/Sources/Workspace/InitPackage.swift @@ -276,7 +276,7 @@ public final class InitPackage { dependencies.append(#".package(url: "https://github.com/swiftlang/swift-syntax.git", from: "\#(self.installedSwiftPMConfiguration.swiftSyntaxVersionForMacroTemplate.description)")"#) } if options.supportedTestingLibraries.contains(.swiftTesting) { - dependencies.append(#".package(url: "https://github.com/swiftlang/swift-testing.git", from: "0.2.0")"#) + dependencies.append(#".package(url: "https://github.com/apple/swift-testing.git", from: "0.11.0")"#) } if !dependencies.isEmpty { let dependencies = dependencies.map { dependency in diff --git a/Sources/XCBuildSupport/XcodeBuildSystem.swift b/Sources/XCBuildSupport/XcodeBuildSystem.swift index 5b8803a0481..bee340282e8 100644 --- a/Sources/XCBuildSupport/XcodeBuildSystem.swift +++ b/Sources/XCBuildSupport/XcodeBuildSystem.swift @@ -58,8 +58,7 @@ public final class XcodeBuildSystem: SPMBuildCore.BuildSystem { BuiltTestProduct( productName: product.name, binaryPath: binaryPath, - packagePath: package.path, - library: buildParameters.testingParameters.library + packagePath: package.path ) ) } diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 6ed336fcbcb..05fe87b93a4 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -1144,17 +1144,16 @@ final class BuildPlanTests: XCTestCase { )) XCTAssertEqual(Set(result.productMap.keys.map(\.productName)), ["APackageTests"]) - #if os(macOS) - XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), ["ATarget", "BTarget", "ATargetTests"]) - #else - XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), [ + var expectedTargets: Set = [ "APackageTests", - "APackageDiscoveredTests", "ATarget", "ATargetTests", "BTarget", - ]) - #endif + ] +#if !os(macOS) + expectedTargets.insert("APackageDiscoveredTests") +#endif + XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), expectedTargets) } func testBasicReleasePackage() throws { @@ -2211,13 +2210,7 @@ final class BuildPlanTests: XCTestCase { observabilityScope: observability.topScope )) result.checkProductsCount(1) - #if os(macOS) - result.checkTargetsCount(2) - #else - // On non-Apple platforms, when a custom entry point file is present (e.g. XCTMain.swift), there is one - // additional target for the synthesized test entry point. result.checkTargetsCount(3) - #endif let buildPath = result.plan.productsBuildPath @@ -2286,6 +2279,8 @@ final class BuildPlanTests: XCTestCase { buildPath.appending(components: "Modules", "Foo.swiftmodule").pathString, "-Xlinker", "-add_ast_path", "-Xlinker", buildPath.appending(components: "Modules", "FooTests.swiftmodule").pathString, + "-Xlinker", "-add_ast_path", "-Xlinker", + buildPath.appending(components: "Modules", "PkgPackageTests.swiftmodule").pathString, "-g", ] ) diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 4fde584289c..41f91562ef8 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -197,7 +197,6 @@ final class TestCommandTests: CommandsTestCase { XCTAssertNoMatch(stdout, .contains("testExample2")) XCTAssertNoMatch(stdout, .contains("testExample3")) XCTAssertNoMatch(stdout, .contains("testExample4")) - XCTAssertMatch(stderr, .contains("No matching test cases were run")) } } diff --git a/Tests/WorkspaceTests/InitTests.swift b/Tests/WorkspaceTests/InitTests.swift index b2f46e2f2f7..96958148cfc 100644 --- a/Tests/WorkspaceTests/InitTests.swift +++ b/Tests/WorkspaceTests/InitTests.swift @@ -179,7 +179,7 @@ final class InitTests: XCTestCase { XCTAssertMatch(manifestContents, .contains(#".tvOS(.v13)"#)) XCTAssertMatch(manifestContents, .contains(#".watchOS(.v6)"#)) XCTAssertMatch(manifestContents, .contains(#".macCatalyst(.v13)"#)) - XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.2.0""#)) + XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.11.0""#)) XCTAssertMatch(manifestContents, .contains(#".product(name: "Testing", package: "swift-testing")"#)) let testFile = path.appending("Tests").appending("FooTests").appending("FooTests.swift") @@ -222,7 +222,7 @@ final class InitTests: XCTestCase { XCTAssertMatch(manifestContents, .contains(#".tvOS(.v13)"#)) XCTAssertMatch(manifestContents, .contains(#".watchOS(.v6)"#)) XCTAssertMatch(manifestContents, .contains(#".macCatalyst(.v13)"#)) - XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.2.0""#)) + XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.11.0""#)) XCTAssertMatch(manifestContents, .contains(#".product(name: "Testing", package: "swift-testing")"#)) let testFile = path.appending("Tests").appending("FooTests").appending("FooTests.swift") @@ -262,7 +262,7 @@ final class InitTests: XCTestCase { let manifest = path.appending("Package.swift") XCTAssertFileExists(manifest) let manifestContents: String = try localFileSystem.readFileContents(manifest) - XCTAssertNoMatch(manifestContents, .contains(#"swift-testing.git", from: "0.2.0""#)) + XCTAssertNoMatch(manifestContents, .contains(#"swift-testing.git", from: "0.11.0""#)) XCTAssertNoMatch(manifestContents, .contains(#".product(name: "Testing", package: "swift-testing")"#)) XCTAssertNoMatch(manifestContents, .contains(#".testTarget"#)) From c86a1e5568774cec838ba0bdeefeb1bcd2e15136 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Tue, 16 Jul 2024 21:36:15 -0400 Subject: [PATCH 3/4] Disable Swift Testing by default in `swift package init`. This PR turns Swift Testing back off by default when running `swift package init` because it's not available in the toolchain yet and the package dependency may come as a surprise (also, it breaks CI, but who's counting?) Follow-up to #7766. --- Sources/Commands/PackageCommands/Init.swift | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Sources/Commands/PackageCommands/Init.swift b/Sources/Commands/PackageCommands/Init.swift index 8fd9522df7e..43903533508 100644 --- a/Sources/Commands/PackageCommands/Init.swift +++ b/Sources/Commands/PackageCommands/Init.swift @@ -55,10 +55,22 @@ extension SwiftPackageCommand { } let packageName = self.packageName ?? cwd.basename + + // Which testing libraries should be used? XCTest is on by default, + // but Swift Testing must remain off by default until it is present + // in the Swift toolchain. + var supportedTestingLibraries = Set() + if testLibraryOptions.isEnabled(.xctest) { + supportedTestingLibraries.insert(.xctest) + } + if testLibraryOptions.explicitlyEnableSwiftTestingLibrarySupport == true || testLibraryOptions.explicitlyEnableExperimentalSwiftTestingLibrarySupport == true { + supportedTestingLibraries.insert(.swiftTesting) + } + let initPackage = try InitPackage( name: packageName, packageType: initMode, - supportedTestingLibraries: Set(testLibraryOptions.enabledTestingLibraries), + supportedTestingLibraries: supportedTestingLibraries, destinationPath: cwd, installedSwiftPMConfiguration: swiftCommandState.getHostToolchain().installedSwiftPMConfiguration, fileSystem: swiftCommandState.fileSystem From 8ac78936f4c4d623e22c25ebaf3d49360e97d639 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 17 Jul 2024 09:13:44 -0400 Subject: [PATCH 4/4] Remove `--enable-swift-testing` _et al._ from `swift build`. The enable/disable testing library options are no longer used by `swift build` and can be removed. --- Sources/Commands/SwiftBuildCommand.swift | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/Sources/Commands/SwiftBuildCommand.swift b/Sources/Commands/SwiftBuildCommand.swift index 1a0ce6b23a0..88cd75c273d 100644 --- a/Sources/Commands/SwiftBuildCommand.swift +++ b/Sources/Commands/SwiftBuildCommand.swift @@ -100,21 +100,6 @@ struct BuildCommandOptions: ParsableArguments { /// If should link the Swift stdlib statically. @Flag(name: .customLong("static-swift-stdlib"), inversion: .prefixedNo, help: "Link Swift stdlib statically") public var shouldLinkStaticSwiftStdlib: Bool = false - - /// Which testing libraries to use (and any related options.) - @OptionGroup() - var testLibraryOptions: TestLibraryOptions - - func validate() throws { - // If --build-tests was not specified, it does not make sense to enable - // or disable either testing library. - if !buildTests { - if testLibraryOptions.explicitlyEnableXCTestSupport != nil - || testLibraryOptions.explicitlyEnableSwiftTestingLibrarySupport != nil { - throw StringError("pass --build-tests to build test targets") - } - } - } } /// swift-build command namespace