diff --git a/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift b/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift index a8adc3923e6..7a0c293105d 100644 --- a/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift +++ b/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift @@ -112,9 +112,7 @@ public class LLBuildManifestBuilder { } } - if self.plan.destinationBuildParameters.testingParameters.library == .xctest { - try self.addTestDiscoveryGenerationCommand() - } + try self.addTestDiscoveryGenerationCommand() try self.addTestEntryPointGenerationCommand() // Create command for all products in the plan. @@ -310,9 +308,7 @@ extension LLBuildManifestBuilder { let outputs = testEntryPointTarget.target.sources.paths - let mainFileName = TestEntryPointTool.mainFileName( - for: self.plan.destinationBuildParameters.testingParameters.library - ) + let mainFileName = TestEntryPointTool.mainFileName guard let mainOutput = (outputs.first { $0.basename == mainFileName }) else { throw InternalError("main output (\(mainFileName)) not found") } diff --git a/Sources/Build/BuildPlan/BuildPlan+Product.swift b/Sources/Build/BuildPlan/BuildPlan+Product.swift index 3967aef58f8..9782103e689 100644 --- a/Sources/Build/BuildPlan/BuildPlan+Product.swift +++ b/Sources/Build/BuildPlan/BuildPlan+Product.swift @@ -275,10 +275,8 @@ extension BuildPlan { } // Add derived test targets, if necessary - if buildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets { - if product.type == .test, let derivedTestTargets = derivedTestTargetsMap[product.id] { - staticTargets.append(contentsOf: derivedTestTargets) - } + if product.type == .test, let derivedTestTargets = derivedTestTargetsMap[product.id] { + staticTargets.append(contentsOf: derivedTestTargets) } return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths, providedLibraries, availableTools) diff --git a/Sources/Build/BuildPlan/BuildPlan+Test.swift b/Sources/Build/BuildPlan/BuildPlan+Test.swift index 53cce82bc97..96b20532d8e 100644 --- a/Sources/Build/BuildPlan/BuildPlan+Test.swift +++ b/Sources/Build/BuildPlan/BuildPlan+Test.swift @@ -34,13 +34,12 @@ extension BuildPlan { _ fileSystem: FileSystem, _ observabilityScope: ObservabilityScope ) throws -> [(product: ResolvedProduct, discoveryTargetBuildDescription: SwiftModuleBuildDescription?, entryPointTargetBuildDescription: SwiftModuleBuildDescription)] { - guard destinationBuildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets, - case .entryPointExecutable(let explicitlyEnabledDiscovery, let explicitlySpecifiedPath) = - destinationBuildParameters.testingParameters.testProductStyle - else { - throw InternalError("makeTestManifestTargets should not be used for build plan which does not require additional derived test targets") + var explicitlyEnabledDiscovery = false + var explicitlySpecifiedPath: AbsolutePath? + if case let .entryPointExecutable(caseExplicitlyEnabledDiscovery, caseExplicitlySpecifiedPath) = destinationBuildParameters.testingParameters.testProductStyle { + explicitlyEnabledDiscovery = caseExplicitlyEnabledDiscovery + explicitlySpecifiedPath = caseExplicitlySpecifiedPath } - let isEntryPointPathSpecifiedExplicitly = explicitlySpecifiedPath != nil var isDiscoveryEnabledRedundantly = explicitlyEnabledDiscovery && !isEntryPointPathSpecifiedExplicitly @@ -116,7 +115,7 @@ extension BuildPlan { resolvedTargetDependencies: [ResolvedModule.Dependency] ) throws -> SwiftModuleBuildDescription { let entryPointDerivedDir = destinationBuildParameters.buildPath.appending(components: "\(testProduct.name).derived") - let entryPointMainFileName = TestEntryPointTool.mainFileName(for: destinationBuildParameters.testingParameters.library) + let entryPointMainFileName = TestEntryPointTool.mainFileName let entryPointMainFile = entryPointDerivedDir.appending(component: entryPointMainFileName) let entryPointSources = Sources(paths: [entryPointMainFile], root: entryPointDerivedDir) @@ -153,18 +152,17 @@ extension BuildPlan { let swiftTargetDependencies: [Module.Dependency] let resolvedTargetDependencies: [ResolvedModule.Dependency] - switch destinationBuildParameters.testingParameters.library { - case .xctest: + if destinationBuildParameters.triple.isDarwin() { + discoveryTargets = nil + swiftTargetDependencies = [] + resolvedTargetDependencies = [] + } else { discoveryTargets = try generateDiscoveryTargets() swiftTargetDependencies = [.module(discoveryTargets!.target, conditions: [])] resolvedTargetDependencies = [.module(discoveryTargets!.resolved, conditions: [])] - case .swiftTesting: - discoveryTargets = nil - swiftTargetDependencies = testProduct.modules.map { .module($0.underlying, conditions: []) } - resolvedTargetDependencies = testProduct.modules.map { .module($0, conditions: []) } } - if let entryPointResolvedTarget = testProduct.testEntryPointModule { + if !destinationBuildParameters.triple.isDarwin(), let entryPointResolvedTarget = testProduct.testEntryPointModule { if isEntryPointPathSpecifiedExplicitly || explicitlyEnabledDiscovery { if isEntryPointPathSpecifiedExplicitly { // Allow using the explicitly-specified test entry point target, but still perform test discovery and thus declare a dependency on the discovery modules. diff --git a/Sources/Build/BuildPlan/BuildPlan.swift b/Sources/Build/BuildPlan/BuildPlan.swift index 12504f0dbb1..e2ce60f9a48 100644 --- a/Sources/Build/BuildPlan/BuildPlan.swift +++ b/Sources/Build/BuildPlan/BuildPlan.swift @@ -443,31 +443,29 @@ public class BuildPlan: SPMBuildCore.BuildPlan { } // Plan the derived test targets, if necessary. - if destinationBuildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets { - let derivedTestTargets = try Self.makeDerivedTestTargets( - testProducts: productMap.values.filter { - $0.product.type == .test - }, - destinationBuildParameters: destinationBuildParameters, - toolsBuildParameters: toolsBuildParameters, - shouldDisableSandbox: self.shouldDisableSandbox, - self.fileSystem, - self.observabilityScope - ) - for item in derivedTestTargets { - var derivedTestTargets = [item.entryPointTargetBuildDescription.target] - - targetMap[item.entryPointTargetBuildDescription.target.id] = .swift( - item.entryPointTargetBuildDescription - ) + let derivedTestTargets = try Self.makeDerivedTestTargets( + testProducts: productMap.values.filter { + $0.product.type == .test + }, + destinationBuildParameters: destinationBuildParameters, + toolsBuildParameters: toolsBuildParameters, + shouldDisableSandbox: self.shouldDisableSandbox, + self.fileSystem, + self.observabilityScope + ) + for item in derivedTestTargets { + var derivedTestTargets = [item.entryPointTargetBuildDescription.target] - if let discoveryTargetBuildDescription = item.discoveryTargetBuildDescription { - targetMap[discoveryTargetBuildDescription.target.id] = .swift(discoveryTargetBuildDescription) - derivedTestTargets.append(discoveryTargetBuildDescription.target) - } + targetMap[item.entryPointTargetBuildDescription.target.id] = .swift( + item.entryPointTargetBuildDescription + ) - self.derivedTestTargetsMap[item.product.id] = derivedTestTargets + if let discoveryTargetBuildDescription = item.discoveryTargetBuildDescription { + targetMap[discoveryTargetBuildDescription.target.id] = .swift(discoveryTargetBuildDescription) + derivedTestTargets.append(discoveryTargetBuildDescription.target) } + + self.derivedTestTargetsMap[item.product.id] = derivedTestTargets } self.productMap = productMap.mapValues(\.buildDescription) diff --git a/Sources/Build/LLBuildCommands.swift b/Sources/Build/LLBuildCommands.swift index a414119c236..f083947608b 100644 --- a/Sources/Build/LLBuildCommands.swift +++ b/Sources/Build/LLBuildCommands.swift @@ -50,8 +50,8 @@ extension IndexStore.TestCaseClass.TestMethod { } extension TestEntryPointTool { - public static func mainFileName(for library: BuildParameters.Testing.Library) -> String { - "runner-\(library).swift" + public static var mainFileName: String { + "runner.swift" } } @@ -105,74 +105,76 @@ final class TestDiscoveryCommand: CustomLLBuildCommand, TestBuildCommand { private func execute(fileSystem: Basics.FileSystem, tool: TestDiscoveryTool) throws { let outputs = tool.outputs.compactMap { try? AbsolutePath(validating: $0.name) } - switch self.context.productsBuildParameters.testingParameters.library { - case .swiftTesting: + if case .loadableBundle = context.productsBuildParameters.testingParameters.testProductStyle { + // When building an XCTest bundle, test discovery is handled by the + // test harness process (i.e. this is the Darwin path.) for file in outputs { try fileSystem.writeIfChanged(path: file, string: "") } - case .xctest: - let index = self.context.productsBuildParameters.indexStore - let api = try self.context.indexStoreAPI.get() - let store = try IndexStore.open(store: TSCAbsolutePath(index), api: api) - - // FIXME: We can speed this up by having one llbuild command per object file. - let tests = try store - .listTests(in: tool.inputs.map { try TSCAbsolutePath(AbsolutePath(validating: $0.name)) }) - - let testsByModule = Dictionary(grouping: tests, by: { $0.module.spm_mangledToC99ExtendedIdentifier() }) - - // Find the main file path. - guard let mainFile = outputs.first(where: { path in - path.basename == TestDiscoveryTool.mainFileName - }) else { - throw InternalError("main output (\(TestDiscoveryTool.mainFileName)) not found") - } + return + } - // Write one file for each test module. - // - // We could write everything in one file but that can easily run into type conflicts due - // in complex packages with large number of test modules. - for file in outputs where file != mainFile { - // FIXME: This is relying on implementation detail of the output but passing the - // the context all the way through is not worth it right now. - let module = file.basenameWithoutExt.spm_mangledToC99ExtendedIdentifier() - - guard let tests = testsByModule[module] else { - // This module has no tests so just write an empty file for it. - try fileSystem.writeFileContents(file, bytes: "") - continue - } - try write( - tests: tests, - forModule: module, - fileSystem: fileSystem, - path: file - ) + let index = self.context.productsBuildParameters.indexStore + let api = try self.context.indexStoreAPI.get() + let store = try IndexStore.open(store: TSCAbsolutePath(index), api: api) + + // FIXME: We can speed this up by having one llbuild command per object file. + let tests = try store + .listTests(in: tool.inputs.map { try TSCAbsolutePath(AbsolutePath(validating: $0.name)) }) + + let testsByModule = Dictionary(grouping: tests, by: { $0.module.spm_mangledToC99ExtendedIdentifier() }) + + // Find the main file path. + guard let mainFile = outputs.first(where: { path in + path.basename == TestDiscoveryTool.mainFileName + }) else { + throw InternalError("main output (\(TestDiscoveryTool.mainFileName)) not found") + } + + // Write one file for each test module. + // + // We could write everything in one file but that can easily run into type conflicts due + // in complex packages with large number of test modules. + for file in outputs where file != mainFile { + // FIXME: This is relying on implementation detail of the output but passing the + // the context all the way through is not worth it right now. + let module = file.basenameWithoutExt.spm_mangledToC99ExtendedIdentifier() + + guard let tests = testsByModule[module] else { + // This module has no tests so just write an empty file for it. + try fileSystem.writeFileContents(file, bytes: "") + continue } + try write( + tests: tests, + forModule: module, + fileSystem: fileSystem, + path: file + ) + } - let testsKeyword = tests.isEmpty ? "let" : "var" + let testsKeyword = tests.isEmpty ? "let" : "var" - // Write the main file. - let stream = try LocalFileOutputByteStream(mainFile) + // Write the main file. + let stream = try LocalFileOutputByteStream(mainFile) - stream.send( - #""" - import XCTest + stream.send( + #""" + import XCTest - @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") - @MainActor - public func __allDiscoveredTests() -> [XCTestCaseEntry] { - \#(testsKeyword) tests = [XCTestCaseEntry]() + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") + @MainActor + public func __allDiscoveredTests() -> [XCTestCaseEntry] { + \#(testsKeyword) tests = [XCTestCaseEntry]() - \#(testsByModule.keys.map { "tests += __\($0)__allTests()" }.joined(separator: "\n ")) + \#(testsByModule.keys.map { "tests += __\($0)__allTests()" }.joined(separator: "\n ")) - return tests - } - """# - ) + return tests + } + """# + ) - stream.flush() - } + stream.flush() } override func execute( @@ -201,9 +203,7 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { let outputs = tool.outputs.compactMap { try? AbsolutePath(validating: $0.name) } // Find the main output file - let mainFileName = TestEntryPointTool.mainFileName( - for: self.context.productsBuildParameters.testingParameters.library - ) + let mainFileName = TestEntryPointTool.mainFileName guard let mainFile = outputs.first(where: { path in path.basename == mainFileName }) else { @@ -213,62 +213,100 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { // Write the main file. let stream = try LocalFileOutputByteStream(mainFile) - switch self.context.productsBuildParameters.testingParameters.library { - case .swiftTesting: - stream.send( - #""" - #if canImport(Testing) - import Testing - #endif + // Find the inputs, which are the names of the test discovery module(s) + let inputs = tool.inputs.compactMap { try? AbsolutePath(validating: $0.name) } + let discoveryModuleNames = inputs.map(\.basenameWithoutExt) - @main struct Runner { - static func main() async { - #if canImport(Testing) - await Testing.__swiftPMEntryPoint() as Never - #endif + let testObservabilitySetup: String + let buildParameters = self.context.productsBuildParameters + if buildParameters.testingParameters.experimentalTestOutput && buildParameters.triple.supportsTestSummary { + testObservabilitySetup = "_ = SwiftPMXCTestObserver()\n" + } else { + testObservabilitySetup = "" + } + + let isXCTMainAvailable: String = switch buildParameters.testingParameters.testProductStyle { + case .entryPointExecutable: + "canImport(XCTest)" + case .loadableBundle: + "false" + } + + /// On WASI, we can't block the main thread, so XCTestMain is defined as async. + let awaitXCTMainKeyword = if buildParameters.triple.isWASI() { + "await" + } else { + "" + } + + var needsAsyncMainWorkaround = false + if buildParameters.triple.isLinux() { + // FIXME: work around crash on Amazon Linux 2 when main function is async (rdar://128303921) + needsAsyncMainWorkaround = true + } else if buildParameters.triple.isDarwin() { +#if compiler(<5.10) + // FIXME: work around duplicate async_Main symbols (SEE https://github.com/swiftlang/swift/pull/69113) + needsAsyncMainWorkaround = true +#endif + } + + stream.send( + #""" + #if canImport(Testing) + import Testing + #endif + + #if \#(isXCTMainAvailable) + \#(generateTestObservationCode(buildParameters: buildParameters)) + + import XCTest + \#(discoveryModuleNames.map { "import \($0)" }.joined(separator: "\n")) + #endif + + @main + @available(macOS 10.15, iOS 11, watchOS 4, tvOS 11, *) + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") + struct Runner { + private static func testingLibrary() -> String { + var iterator = CommandLine.arguments.makeIterator() + while let argument = iterator.next() { + if argument == "--testing-library", let libraryName = iterator.next() { + return libraryName.lowercased() + } } - } - """# - ) - case .xctest: - // Find the inputs, which are the names of the test discovery module(s) - let inputs = tool.inputs.compactMap { try? AbsolutePath(validating: $0.name) } - let discoveryModuleNames = inputs.map(\.basenameWithoutExt) - - let testObservabilitySetup: String - let buildParameters = self.context.productsBuildParameters - if buildParameters.testingParameters.experimentalTestOutput && buildParameters.triple.supportsTestSummary { - testObservabilitySetup = "_ = SwiftPMXCTestObserver()\n" - } else { - testObservabilitySetup = "" - } - stream.send( - #""" - \#(generateTestObservationCode(buildParameters: buildParameters)) + // Fallback if not specified: run XCTest (legacy behavior) + return "xctest" + } - import XCTest - \#(discoveryModuleNames.map { "import \($0)" }.joined(separator: "\n")) + #if \#(needsAsyncMainWorkaround) + @_silgen_name("$ss13_runAsyncMainyyyyYaKcF") + private static func _runAsyncMain(_ asyncFun: @Sendable @escaping () async throws -> ()) + #endif - @main - @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") - struct Runner { - #if os(WASI) - /// On WASI, we can't block the main thread, so XCTestMain is defined as async. - static func main() async { - \#(testObservabilitySetup) - await XCTMain(__allDiscoveredTests()) as Never + static func main() \#(needsAsyncMainWorkaround ? "" : "async") { + let testingLibrary = Self.testingLibrary() + #if canImport(Testing) + if testingLibrary == "swift-testing" { + #if \#(needsAsyncMainWorkaround) + _runAsyncMain { + await Testing.__swiftPMEntryPoint() as Never + } + #else + await Testing.__swiftPMEntryPoint() as Never + #endif } - #else - static func main() { + #endif + #if \#(isXCTMainAvailable) + if testingLibrary == "xctest" { \#(testObservabilitySetup) - XCTMain(__allDiscoveredTests()) as Never + \#(awaitXCTMainKeyword) XCTMain(__allDiscoveredTests()) as Never } #endif } - """# - ) - } + } + """# + ) stream.flush() } diff --git a/Sources/Build/LLBuildDescription.swift b/Sources/Build/LLBuildDescription.swift index 5612d33918e..bafa740ba1c 100644 --- a/Sources/Build/LLBuildDescription.swift +++ b/Sources/Build/LLBuildDescription.swift @@ -111,8 +111,7 @@ public struct BuildDescription: Codable { try BuiltTestProduct( productName: desc.product.name, binaryPath: desc.binaryPath, - packagePath: desc.package.path, - library: desc.buildParameters.testingParameters.library + packagePath: desc.package.path ) } self.pluginDescriptions = pluginDescriptions diff --git a/Sources/Commands/PackageCommands/Init.swift b/Sources/Commands/PackageCommands/Init.swift index 6263e84b835..43903533508 100644 --- a/Sources/Commands/PackageCommands/Init.swift +++ b/Sources/Commands/PackageCommands/Init.swift @@ -54,20 +54,23 @@ extension SwiftPackageCommand { throw InternalError("Could not find the current working directory") } - // NOTE: Do not use testLibraryOptions.enabledTestingLibraries(swiftCommandState:) here - // because the package doesn't exist yet, so there are no dependencies for it to query. - var testingLibraries: Set = [] - if testLibraryOptions.enableXCTestSupport { - testingLibraries.insert(.xctest) + let packageName = self.packageName ?? cwd.basename + + // Which testing libraries should be used? XCTest is on by default, + // but Swift Testing must remain off by default until it is present + // in the Swift toolchain. + var supportedTestingLibraries = Set() + if testLibraryOptions.isEnabled(.xctest) { + supportedTestingLibraries.insert(.xctest) } - if testLibraryOptions.explicitlyEnableSwiftTestingLibrarySupport == true { - testingLibraries.insert(.swiftTesting) + if testLibraryOptions.explicitlyEnableSwiftTestingLibrarySupport == true || testLibraryOptions.explicitlyEnableExperimentalSwiftTestingLibrarySupport == true { + supportedTestingLibraries.insert(.swiftTesting) } - let packageName = self.packageName ?? cwd.basename + let initPackage = try InitPackage( name: packageName, packageType: initMode, - supportedTestingLibraries: testingLibraries, + supportedTestingLibraries: supportedTestingLibraries, destinationPath: cwd, installedSwiftPMConfiguration: swiftCommandState.getHostToolchain().installedSwiftPMConfiguration, fileSystem: swiftCommandState.fileSystem diff --git a/Sources/Commands/SwiftBuildCommand.swift b/Sources/Commands/SwiftBuildCommand.swift index 53c8db111a1..88cd75c273d 100644 --- a/Sources/Commands/SwiftBuildCommand.swift +++ b/Sources/Commands/SwiftBuildCommand.swift @@ -100,21 +100,6 @@ struct BuildCommandOptions: ParsableArguments { /// If should link the Swift stdlib statically. @Flag(name: .customLong("static-swift-stdlib"), inversion: .prefixedNo, help: "Link Swift stdlib statically") public var shouldLinkStaticSwiftStdlib: Bool = false - - /// Which testing libraries to use (and any related options.) - @OptionGroup() - var testLibraryOptions: TestLibraryOptions - - func validate() throws { - // If --build-tests was not specified, it does not make sense to enable - // or disable either testing library. - if !buildTests { - if testLibraryOptions.explicitlyEnableXCTestSupport != nil - || testLibraryOptions.explicitlyEnableSwiftTestingLibrarySupport != nil { - throw StringError("pass --build-tests to build test targets") - } - } - } } /// swift-build command namespace @@ -159,35 +144,12 @@ public struct SwiftBuildCommand: AsyncSwiftCommand { var productsBuildParameters = try swiftCommandState.productsBuildParameters var toolsBuildParameters = try swiftCommandState.toolsBuildParameters - // Clean out the code coverage directory that may contain stale - // profraw files from a previous run of the code coverage tool. if self.options.enableCodeCoverage { - try swiftCommandState.fileSystem.removeFileTree(swiftCommandState.productsBuildParameters.codeCovPath) productsBuildParameters.testingParameters.enableCodeCoverage = true toolsBuildParameters.testingParameters.enableCodeCoverage = true } - if case .allIncludingTests = subset { - func updateTestingParameters(of buildParameters: inout BuildParameters, library: BuildParameters.Testing.Library) { - buildParameters.testingParameters = .init( - configuration: buildParameters.configuration, - targetTriple: buildParameters.triple, - enableCodeCoverage: buildParameters.testingParameters.enableCodeCoverage, - enableTestability: buildParameters.testingParameters.enableTestability, - experimentalTestOutput: buildParameters.testingParameters.experimentalTestOutput, - forceTestDiscovery: globalOptions.build.enableTestDiscovery, - testEntryPointPath: globalOptions.build.testEntryPointPath, - library: library - ) - } - for library in try options.testLibraryOptions.enabledTestingLibraries(swiftCommandState: swiftCommandState) { - updateTestingParameters(of: &productsBuildParameters, library: library) - updateTestingParameters(of: &toolsBuildParameters, library: library) - try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) - } - } else { - try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) - } + try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) } private func build( diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index 8ff59d6d2f0..2902b006de1 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -37,6 +37,10 @@ import var TSCBasic.stdoutStream import class TSCBasic.SynchronizedQueue import class TSCBasic.Thread +#if os(Windows) +import WinSDK // for ERROR_NOT_FOUND +#endif + private enum TestError: Swift.Error { case invalidListTestJSONData(context: String, underlyingError: Error? = nil) case testsNotFound @@ -241,102 +245,116 @@ public struct SwiftTestCommand: AsyncSwiftCommand { @OptionGroup() var options: TestCommandOptions - // MARK: - XCTest - - private func xctestRun(_ swiftCommandState: SwiftCommandState) async throws { - // validate XCTest available on darwin based systems - let toolchain = try swiftCommandState.getTargetToolchain() - if case let .unsupported(reason) = try swiftCommandState.getHostToolchain().swiftSDK.xctestSupport { - if let reason { - throw TestError.xctestNotAvailable(reason: reason) - } else { - throw TestError.xcodeNotInstalled - } - } else if toolchain.targetTriple.isDarwin() && toolchain.xctestPath == nil { - throw TestError.xcodeNotInstalled - } - - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: .xctest) - + private func run(_ swiftCommandState: SwiftCommandState, buildParameters: BuildParameters, testProducts: [BuiltTestProduct]) async throws { // Remove test output from prior runs and validate priors. - if self.options.enableExperimentalTestOutput && productsBuildParameters.triple.supportsTestSummary { - _ = try? localFileSystem.removeFileTree(productsBuildParameters.testOutputPath) + if self.options.enableExperimentalTestOutput && buildParameters.triple.supportsTestSummary { + _ = try? localFileSystem.removeFileTree(buildParameters.testOutputPath) } - let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState, library: .xctest) - if !self.options.shouldRunInParallel { - let xctestArgs = try xctestArgs(for: testProducts, swiftCommandState: swiftCommandState) - try await runTestProducts( - testProducts, - additionalArguments: xctestArgs, - productsBuildParameters: productsBuildParameters, - swiftCommandState: swiftCommandState, - library: .xctest - ) - } else { - let testSuites = try TestingSupport.getTestSuites( - in: testProducts, - swiftCommandState: swiftCommandState, - enableCodeCoverage: options.enableCodeCoverage, - shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, - experimentalTestOutput: options.enableExperimentalTestOutput, - sanitizers: globalOptions.build.sanitizers - ) - let tests = try testSuites - .filteredTests(specifier: options.testCaseSpecifier) - .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) - - // If there were no matches, emit a warning and exit. - if tests.isEmpty { - swiftCommandState.observabilityScope.emit(.noMatchingTests) - try generateXUnitOutputIfRequested(for: [], swiftCommandState: swiftCommandState) - return - } + var results = [TestRunner.Result]() - // Clean out the code coverage directory that may contain stale - // profraw files from a previous run of the code coverage tool. - if self.options.enableCodeCoverage { - try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) + // Run XCTest. + if options.testLibraryOptions.isEnabled(.xctest) { + // validate XCTest available on darwin based systems + let toolchain = try swiftCommandState.getTargetToolchain() + if case let .unsupported(reason) = try swiftCommandState.getHostToolchain().swiftSDK.xctestSupport { + if let reason { + throw TestError.xctestNotAvailable(reason: reason) + } else { + throw TestError.xcodeNotInstalled + } + } else if toolchain.targetTriple.isDarwin() && toolchain.xctestPath == nil { + throw TestError.xcodeNotInstalled } - // Run the tests using the parallel runner. - let runner = ParallelTestRunner( - bundlePaths: testProducts.map { $0.bundlePath }, - cancellator: swiftCommandState.cancellator, - toolchain: toolchain, - numJobs: options.numberOfWorkers ?? ProcessInfo.processInfo.activeProcessorCount, - buildOptions: globalOptions.build, - productsBuildParameters: productsBuildParameters, - shouldOutputSuccess: swiftCommandState.logLevel <= .info, - observabilityScope: swiftCommandState.observabilityScope - ) - - let testResults = try runner.run(tests) + if !self.options.shouldRunInParallel { + let (xctestArgs, testCount) = try xctestArgs(for: testProducts, swiftCommandState: swiftCommandState) + let result = try await runTestProducts( + testProducts, + additionalArguments: xctestArgs, + productsBuildParameters: buildParameters, + swiftCommandState: swiftCommandState, + library: .xctest + ) + if result == .success && testCount == 0 { + results.append(.noMatchingTests) + } else { + results.append(result) + } + } else { + let testSuites = try TestingSupport.getTestSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + enableCodeCoverage: options.enableCodeCoverage, + shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, + experimentalTestOutput: options.enableExperimentalTestOutput, + sanitizers: globalOptions.build.sanitizers + ) + let tests = try testSuites + .filteredTests(specifier: options.testCaseSpecifier) + .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) + + let result: TestRunner.Result + let testResults: [ParallelTestRunner.TestResult] + if tests.isEmpty { + testResults = [] + result = .noMatchingTests + } else { + // Run the tests using the parallel runner. + let runner = ParallelTestRunner( + bundlePaths: testProducts.map { $0.bundlePath }, + cancellator: swiftCommandState.cancellator, + toolchain: toolchain, + numJobs: options.numberOfWorkers ?? ProcessInfo.processInfo.activeProcessorCount, + buildOptions: globalOptions.build, + productsBuildParameters: buildParameters, + shouldOutputSuccess: swiftCommandState.logLevel <= .info, + observabilityScope: swiftCommandState.observabilityScope + ) - try generateXUnitOutputIfRequested(for: testResults, swiftCommandState: swiftCommandState) + testResults = try runner.run(tests) + result = runner.ranSuccessfully ? .success : .failure + } - // Process code coverage if requested - if self.options.enableCodeCoverage, runner.ranSuccessfully { - try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState, library: .xctest) + try generateXUnitOutputIfRequested(for: testResults, swiftCommandState: swiftCommandState) + results.append(result) } + } - if !runner.ranSuccessfully { - swiftCommandState.executionStatus = .failure - } + // Run Swift Testing (parallel or not, it has a single entry point.) + if options.testLibraryOptions.isEnabled(.swiftTesting) { + results.append( + try await runTestProducts( + testProducts, + additionalArguments: [], + productsBuildParameters: buildParameters, + swiftCommandState: swiftCommandState, + library: .swiftTesting + ) + ) + } - if self.options.enableExperimentalTestOutput, !runner.ranSuccessfully { - try Self.handleTestOutput(productsBuildParameters: productsBuildParameters, packagePath: testProducts[0].packagePath) + switch results.reduce() { + case .success: + // Nothing to do here. + break + case .failure: + swiftCommandState.executionStatus = .failure + if self.options.enableExperimentalTestOutput { + try Self.handleTestOutput(productsBuildParameters: buildParameters, packagePath: testProducts[0].packagePath) } + case .noMatchingTests: + swiftCommandState.observabilityScope.emit(.noMatchingTests) } } - private func xctestArgs(for testProducts: [BuiltTestProduct], swiftCommandState: SwiftCommandState) throws -> [String] { + private func xctestArgs(for testProducts: [BuiltTestProduct], swiftCommandState: SwiftCommandState) throws -> (arguments: [String], testCount: Int) { switch options.testCaseSpecifier { case .none: if case .skip = options.skippedTests(fileSystem: swiftCommandState.fileSystem) { fallthrough } else { - return [] + return ([], 0) } case .regex, .specific, .skip: @@ -358,12 +376,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { .filteredTests(specifier: options.testCaseSpecifier) .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) - // If there were no matches, emit a warning. - if tests.isEmpty { - swiftCommandState.observabilityScope.emit(.noMatchingTests) - } - - return TestRunner.xctestArguments(forTestSpecifiers: tests.map(\.specifier)) + return (TestRunner.xctestArguments(forTestSpecifiers: tests.map(\.specifier)), tests.count) } } @@ -383,21 +396,6 @@ public struct SwiftTestCommand: AsyncSwiftCommand { try generator.generate(at: xUnitOutput) } - // MARK: - swift-testing - - private func swiftTestingRun(_ swiftCommandState: SwiftCommandState) async throws { - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: .swiftTesting) - let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState, library: .swiftTesting) - let additionalArguments = Array(CommandLine.arguments.dropFirst()) - try await runTestProducts( - testProducts, - additionalArguments: additionalArguments, - productsBuildParameters: productsBuildParameters, - swiftCommandState: swiftCommandState, - library: .swiftTesting - ) - } - // MARK: - Common implementation public func run(_ swiftCommandState: SwiftCommandState) async throws { @@ -416,12 +414,22 @@ public struct SwiftTestCommand: AsyncSwiftCommand { let command = try List.parse() try command.run(swiftCommandState) } else { - if try options.testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { - try await swiftTestingRun(swiftCommandState) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) + let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState) + + // Clean out the code coverage directory that may contain stale + // profraw files from a previous run of the code coverage tool. + if self.options.enableCodeCoverage { + try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) } - if options.testLibraryOptions.enableXCTestSupport { - try await xctestRun(swiftCommandState) + + try await run(swiftCommandState, buildParameters: productsBuildParameters, testProducts: testProducts) + + // process code Coverage if request + if self.options.enableCodeCoverage, swiftCommandState.executionStatus != .failure { + try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState) } + } } @@ -431,11 +439,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { productsBuildParameters: BuildParameters, swiftCommandState: SwiftCommandState, library: BuildParameters.Testing.Library - ) async throws { - // Clean out the code coverage directory that may contain stale - // profraw files from a previous run of the code coverage tool. - if self.options.enableCodeCoverage { - try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) + ) async throws -> TestRunner.Result { + // Pass through all arguments from the command line to Swift Testing. + var additionalArguments = additionalArguments + if library == .swiftTesting { + additionalArguments += CommandLine.arguments.dropFirst() } let toolchain = try swiftCommandState.getTargetToolchain() @@ -446,8 +454,15 @@ public struct SwiftTestCommand: AsyncSwiftCommand { library: library ) + let runnerPaths: [AbsolutePath] = switch library { + case .xctest: + testProducts.map(\.bundlePath) + case .swiftTesting: + testProducts.map(\.binaryPath) + } + let runner = TestRunner( - bundlePaths: testProducts.map { library == .xctest ? $0.bundlePath : $0.binaryPath }, + bundlePaths: runnerPaths, additionalArguments: additionalArguments, cancellator: swiftCommandState.cancellator, toolchain: toolchain, @@ -457,22 +472,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { ) // Finally, run the tests. - let ranSuccessfully = runner.test(outputHandler: { + return runner.test(outputHandler: { // command's result output goes on stdout // ie "swift test" should output to stdout print($0, terminator: "") }) - if !ranSuccessfully { - swiftCommandState.executionStatus = .failure - } - - if self.options.enableCodeCoverage, ranSuccessfully { - try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState, library: library) - } - - if self.options.enableExperimentalTestOutput, !ranSuccessfully { - try Self.handleTestOutput(productsBuildParameters: productsBuildParameters, packagePath: testProducts[0].packagePath) - } } private static func handleTestOutput(productsBuildParameters: BuildParameters, packagePath: AbsolutePath) throws { @@ -509,8 +513,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { /// Processes the code coverage data and emits a json. private func processCodeCoverage( _ testProducts: [BuiltTestProduct], - swiftCommandState: SwiftCommandState, - library: BuildParameters.Testing.Library + swiftCommandState: SwiftCommandState ) async throws { let workspace = try swiftCommandState.getActiveWorkspace() let root = try swiftCommandState.getWorkspaceRoot() @@ -523,23 +526,23 @@ public struct SwiftTestCommand: AsyncSwiftCommand { } // Merge all the profraw files to produce a single profdata file. - try mergeCodeCovRawDataFiles(swiftCommandState: swiftCommandState, library: library) + try mergeCodeCovRawDataFiles(swiftCommandState: swiftCommandState) - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) for product in testProducts { // Export the codecov data as JSON. let jsonPath = productsBuildParameters.codeCovAsJSONPath(packageName: rootManifest.displayName) - try exportCodeCovAsJSON(to: jsonPath, testBinary: product.binaryPath, swiftCommandState: swiftCommandState, library: library) + try exportCodeCovAsJSON(to: jsonPath, testBinary: product.binaryPath, swiftCommandState: swiftCommandState) } } /// Merges all profraw profiles in codecoverage directory into default.profdata file. - private func mergeCodeCovRawDataFiles(swiftCommandState: SwiftCommandState, library: BuildParameters.Testing.Library) throws { + private func mergeCodeCovRawDataFiles(swiftCommandState: SwiftCommandState) throws { // Get the llvm-prof tool. let llvmProf = try swiftCommandState.getTargetToolchain().getLLVMProf() // Get the profraw files. - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) let codeCovFiles = try swiftCommandState.fileSystem.getDirectoryContents(productsBuildParameters.codeCovPath) // Construct arguments for invoking the llvm-prof tool. @@ -559,12 +562,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { private func exportCodeCovAsJSON( to path: AbsolutePath, testBinary: AbsolutePath, - swiftCommandState: SwiftCommandState, - library: BuildParameters.Testing.Library + swiftCommandState: SwiftCommandState ) throws { // Export using the llvm-cov tool. let llvmCov = try swiftCommandState.getTargetToolchain().getLLVMCov() - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) let args = [ llvmCov.pathString, "export", @@ -584,10 +586,9 @@ public struct SwiftTestCommand: AsyncSwiftCommand { /// /// - Returns: The paths to the build test products. private func buildTestsIfNeeded( - swiftCommandState: SwiftCommandState, - library: BuildParameters.Testing.Library + swiftCommandState: SwiftCommandState ) throws -> [BuiltTestProduct] { - let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) + let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest(options: self.options) return try Commands.buildTestsIfNeeded( swiftCommandState: swiftCommandState, productsBuildParameters: productsBuildParameters, @@ -614,7 +615,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { throw StringError("'--num-workers' must be greater than zero") } - if !options.testLibraryOptions.enableXCTestSupport { + guard options.testLibraryOptions.isEnabled(.xctest) else { throw StringError("'--num-workers' is only supported when testing with XCTest") } } @@ -641,7 +642,7 @@ extension SwiftTestCommand { guard let rootManifest = rootManifests.values.first else { throw StringError("invalid manifests at \(root.packages)") } - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(enableCodeCoverage: true, library: .xctest) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(enableCodeCoverage: true) print(productsBuildParameters.codeCovAsJSONPath(packageName: rootManifest.displayName)) } } @@ -682,41 +683,10 @@ extension SwiftTestCommand { @Flag(name: [.customLong("list-tests"), .customShort("l")], help: .hidden) var _deprecated_passthrough: Bool = false - // MARK: - XCTest - - private func xctestRun(_ swiftCommandState: SwiftCommandState) throws { - let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest( - enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding, - library: .xctest - ) - let testProducts = try buildTestsIfNeeded( - swiftCommandState: swiftCommandState, - productsBuildParameters: productsBuildParameters, - toolsBuildParameters: toolsBuildParameters - ) - let testSuites = try TestingSupport.getTestSuites( - in: testProducts, - swiftCommandState: swiftCommandState, - enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding, - experimentalTestOutput: false, - sanitizers: globalOptions.build.sanitizers - ) - - // Print the tests. - for test in testSuites.allTests { - print(test.specifier) - } - } - - // MARK: - swift-testing - - private func swiftTestingRun(_ swiftCommandState: SwiftCommandState) throws { + func run(_ swiftCommandState: SwiftCommandState) throws { let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest( enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding, - library: .swiftTesting + shouldSkipBuilding: sharedOptions.shouldSkipBuilding ) let testProducts = try buildTestsIfNeeded( swiftCommandState: swiftCommandState, @@ -732,36 +702,43 @@ extension SwiftTestCommand { library: .swiftTesting ) - let additionalArguments = ["--list-tests"] + CommandLine.arguments.dropFirst() - let runner = TestRunner( - bundlePaths: testProducts.map(\.binaryPath), - additionalArguments: additionalArguments, - cancellator: swiftCommandState.cancellator, - toolchain: toolchain, - testEnv: testEnv, - observabilityScope: swiftCommandState.observabilityScope, - library: .swiftTesting - ) - - // Finally, run the tests. - let ranSuccessfully = runner.test(outputHandler: { - // command's result output goes on stdout - // ie "swift test" should output to stdout - print($0, terminator: "") - }) - if !ranSuccessfully { - swiftCommandState.executionStatus = .failure + if testLibraryOptions.isEnabled(.xctest) { + let testSuites = try TestingSupport.getTestSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + enableCodeCoverage: false, + shouldSkipBuilding: sharedOptions.shouldSkipBuilding, + experimentalTestOutput: false, + sanitizers: globalOptions.build.sanitizers + ) + + // Print the tests. + for test in testSuites.allTests { + print(test.specifier) + } } - } - - // MARK: - Common implementation - func run(_ swiftCommandState: SwiftCommandState) throws { - if try testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { - try swiftTestingRun(swiftCommandState) - } - if testLibraryOptions.enableXCTestSupport { - try xctestRun(swiftCommandState) + if testLibraryOptions.isEnabled(.swiftTesting) { + let additionalArguments = ["--list-tests"] + CommandLine.arguments.dropFirst() + let runner = TestRunner( + bundlePaths: testProducts.map(\.binaryPath), + additionalArguments: additionalArguments, + cancellator: swiftCommandState.cancellator, + toolchain: toolchain, + testEnv: testEnv, + observabilityScope: swiftCommandState.observabilityScope, + library: .swiftTesting + ) + + // Finally, run the tests. + let result = runner.test(outputHandler: { + // command's result output goes on stdout + // ie "swift test" should output to stdout + print($0, terminator: "") + }) + if result == .failure { + swiftCommandState.executionStatus = .failure + } } } @@ -860,39 +837,62 @@ final class TestRunner { self.library = library } + /// The result of running the test(s). + enum Result: Equatable { + /// The test(s) ran successfully. + case success + + /// The test(s) failed. + case failure + + /// There were no matching tests to run. + /// + /// XCTest does not report this result. It is used by Swift Testing only. + case noMatchingTests + } + /// Executes and returns execution status. Prints test output on standard streams if requested - /// - Returns: Boolean indicating if test execution returned code 0, and the output stream result - public func test(outputHandler: @escaping (String) -> Void) -> Bool { - var success = true + /// - Returns: Result of spawning and running the test process, and the output stream result + func test(outputHandler: @escaping (String) -> Void) -> Result { + var results = [Result]() for path in self.bundlePaths { let testSuccess = self.test(at: path, outputHandler: outputHandler) - success = success && testSuccess + results.append(testSuccess) } - return success + return results.reduce() } /// Constructs arguments to execute XCTest. private func args(forTestAt testPath: AbsolutePath) throws -> [String] { var args: [String] = [] - #if os(macOS) - if library == .xctest { +#if os(macOS) + switch library { + case .xctest: guard let xctestPath = self.toolchain.xctestPath else { throw TestError.xcodeNotInstalled } - args = [xctestPath.pathString] - args += additionalArguments - args += [testPath.pathString] - return args + args += [xctestPath.pathString] + case .swiftTesting: + let helper = try self.toolchain.getSwiftTestingHelper() + args += [helper.pathString, "--test-bundle-path", testPath.pathString] } - #endif - - args += [testPath.description] args += additionalArguments + args += [testPath.pathString] +#else + args += [testPath.pathString] + args += additionalArguments +#endif + + if library == .swiftTesting { + // HACK: tell the test bundle/executable that we want to run Swift Testing, not XCTest. + // XCTest doesn't understand this argument (yet), so don't pass it there. + args += ["--testing-library", "swift-testing"] + } return args } - private func test(at path: AbsolutePath, outputHandler: @escaping (String) -> Void) -> Bool { + private func test(at path: AbsolutePath, outputHandler: @escaping (String) -> Void) -> Result { let testObservabilityScope = self.observabilityScope.makeChildScope(description: "running test at \(path)") do { @@ -907,25 +907,40 @@ final class TestRunner { ) let process = AsyncProcess(arguments: try args(forTestAt: path), environment: self.testEnv, outputRedirection: outputRedirection) guard let terminationKey = self.cancellator.register(process) else { - return false // terminating + return .failure // terminating } defer { self.cancellator.deregister(terminationKey) } try process.launch() let result = try process.waitUntilExit() switch result.exitStatus { case .terminated(code: 0): - return true + return .success + case .terminated(code: EXIT_NO_TESTS_FOUND) where library == .swiftTesting: + return .noMatchingTests #if !os(Windows) case .signalled(let signal) where ![SIGINT, SIGKILL, SIGTERM].contains(signal): testObservabilityScope.emit(error: "Exited with unexpected signal code \(signal)") - return false + return .failure #endif default: - return false + return .failure } } catch { testObservabilityScope.emit(error) - return false + return .failure + } + } +} + +extension Collection where Element == TestRunner.Result { + /// Reduce all results in this collection into a single result. + func reduce() -> Element { + if contains(.failure) { + return .failure + } else if isEmpty || contains(.success) { + return .success + } else { + return .noMatchingTests } } } @@ -1061,20 +1076,20 @@ final class ParallelTestRunner { toolchain: self.toolchain, testEnv: testEnv, observabilityScope: self.observabilityScope, - library: .xctest + library: .xctest // swift-testing does not use ParallelTestRunner ) var output = "" let outputLock = NSLock() let start = DispatchTime.now() - let success = testRunner.test(outputHandler: { _output in outputLock.withLock{ output += _output }}) + let result = testRunner.test(outputHandler: { _output in outputLock.withLock{ output += _output }}) let duration = start.distance(to: .now()) - if !success { + if result == .failure { self.ranSuccessfully = false } self.finishedTests.enqueue(TestResult( unitTest: test, output: output, - success: success, + success: result != .failure, duration: duration )) } @@ -1325,21 +1340,14 @@ final class XUnitGenerator { extension SwiftCommandState { func buildParametersForTest( - options: TestCommandOptions, - library: BuildParameters.Testing.Library + options: TestCommandOptions ) throws -> (productsBuildParameters: BuildParameters, toolsBuildParameters: BuildParameters) { - var result = try self.buildParametersForTest( + try self.buildParametersForTest( enableCodeCoverage: options.enableCodeCoverage, enableTestability: options.enableTestableImports, shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, - experimentalTestOutput: options.enableExperimentalTestOutput, - library: library + experimentalTestOutput: options.enableExperimentalTestOutput ) - if try options.testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: self) { - result.productsBuildParameters.flags.swiftCompilerFlags += ["-DSWIFT_PM_SUPPORTS_SWIFT_TESTING"] - result.toolsBuildParameters.flags.swiftCompilerFlags += ["-DSWIFT_PM_SUPPORTS_SWIFT_TESTING"] - } - return result } } @@ -1392,6 +1400,24 @@ private extension Basics.Diagnostic { } } +/// The exit code returned to Swift Package Manager by Swift Testing when no +/// tests matched the inputs specified by the developer (or, for the case of +/// `swift test list`, when no tests were found.) +/// +/// Because Swift Package Manager does not directly link to the testing library, +/// it duplicates the definition of this constant in its own source. Any changes +/// to this constant in either package must be mirrored in the other. +private var EXIT_NO_TESTS_FOUND: CInt { +#if os(macOS) || os(Linux) + EX_UNAVAILABLE +#elseif os(Windows) + ERROR_NOT_FOUND +#else +#warning("Platform-specific implementation missing: value for EXIT_NO_TESTS_FOUND unavailable") + return 2 // We're assuming that EXIT_SUCCESS = 0 and EXIT_FAILURE = 1. +#endif +} + /// Builds the "test" target if enabled in options. /// /// - Returns: The paths to the build test products. diff --git a/Sources/Commands/Utilities/PluginDelegate.swift b/Sources/Commands/Utilities/PluginDelegate.swift index a118930abec..94c6bf3fffe 100644 --- a/Sources/Commands/Utilities/PluginDelegate.swift +++ b/Sources/Commands/Utilities/PluginDelegate.swift @@ -284,11 +284,11 @@ final class PluginDelegate: PluginInvocationDelegate { // Run the test — for now we run the sequentially so we can capture accurate timing results. let startTime = DispatchTime.now() - let success = testRunner.test(outputHandler: { _ in }) // this drops the tests output + let result = testRunner.test(outputHandler: { _ in }) // this drops the tests output let duration = Double(startTime.distance(to: .now()).milliseconds() ?? 0) / 1000.0 - numFailedTests += success ? 0 : 1 + numFailedTests += (result != .failure) ? 0 : 1 testResults.append( - .init(name: testName, result: success ? .succeeded : .failed, duration: duration) + .init(name: testName, result: (result != .failure) ? .succeeded : .failed, duration: duration) ) } diff --git a/Sources/Commands/Utilities/TestingSupport.swift b/Sources/Commands/Utilities/TestingSupport.swift index 59c7246e0a3..1291ac73e43 100644 --- a/Sources/Commands/Utilities/TestingSupport.swift +++ b/Sources/Commands/Utilities/TestingSupport.swift @@ -118,8 +118,7 @@ enum TestingSupport { buildParameters: swiftCommandState.buildParametersForTest( enableCodeCoverage: enableCodeCoverage, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput, - library: .xctest + experimentalTestOutput: experimentalTestOutput ).productsBuildParameters, sanitizers: sanitizers, library: .xctest @@ -134,8 +133,7 @@ enum TestingSupport { toolchain: try swiftCommandState.getTargetToolchain(), buildParameters: swiftCommandState.buildParametersForTest( enableCodeCoverage: enableCodeCoverage, - shouldSkipBuilding: shouldSkipBuilding, - library: .xctest + shouldSkipBuilding: shouldSkipBuilding ).productsBuildParameters, sanitizers: sanitizers, library: .xctest @@ -164,10 +162,6 @@ enum TestingSupport { env["NO_COLOR"] = "1" } - // Set an environment variable to indicate which library's test product - // is being executed. - env["SWIFT_PM_TEST_LIBRARY"] = String(describing: library) - // Add the code coverage related variables. if buildParameters.testingParameters.enableCodeCoverage { // Defines the path at which the profraw files will be written on test execution. @@ -177,7 +171,7 @@ enum TestingSupport { // execution but is required when the tests are running in parallel as // SwiftPM repeatedly invokes the test binary with the test case name as // the filter. - let codecovProfile = buildParameters.buildPath.appending(components: "codecov", "default%m.profraw") + let codecovProfile = buildParameters.buildPath.appending(components: "codecov", "\(library)%m.profraw") env["LLVM_PROFILE_FILE"] = codecovProfile.pathString } #if !os(macOS) @@ -195,6 +189,11 @@ enum TestingSupport { env.appendPath(key: "DYLD_LIBRARY_PATH", value: sdkPlatformFrameworksPath.lib.pathString) } + // We aren't using XCTest's harness logic to run Swift Testing tests. + if library == .xctest { + env["SWIFT_TESTING_ENABLED"] = "0" + } + // Fast path when no sanitizers are enabled. if sanitizers.isEmpty { return env @@ -221,24 +220,21 @@ extension SwiftCommandState { enableCodeCoverage: Bool, enableTestability: Bool? = nil, shouldSkipBuilding: Bool = false, - experimentalTestOutput: Bool = false, - library: BuildParameters.Testing.Library + experimentalTestOutput: Bool = false ) throws -> (productsBuildParameters: BuildParameters, toolsBuildParameters: BuildParameters) { let productsBuildParameters = buildParametersForTest( modifying: try productsBuildParameters, enableCodeCoverage: enableCodeCoverage, enableTestability: enableTestability, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput, - library: library + experimentalTestOutput: experimentalTestOutput ) let toolsBuildParameters = buildParametersForTest( modifying: try toolsBuildParameters, enableCodeCoverage: enableCodeCoverage, enableTestability: enableTestability, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput, - library: library + experimentalTestOutput: experimentalTestOutput ) return (productsBuildParameters, toolsBuildParameters) } @@ -248,8 +244,7 @@ extension SwiftCommandState { enableCodeCoverage: Bool, enableTestability: Bool?, shouldSkipBuilding: Bool, - experimentalTestOutput: Bool, - library: BuildParameters.Testing.Library + experimentalTestOutput: Bool ) -> BuildParameters { var parameters = parameters @@ -266,8 +261,7 @@ extension SwiftCommandState { configuration: parameters.configuration, targetTriple: parameters.triple, forceTestDiscovery: explicitlyEnabledDiscovery, - testEntryPointPath: explicitlySpecifiedPath, - library: library + testEntryPointPath: explicitlySpecifiedPath ) parameters.testingParameters.enableCodeCoverage = enableCodeCoverage diff --git a/Sources/CoreCommands/Options.swift b/Sources/CoreCommands/Options.swift index 20546383c09..eb6dd14a868 100644 --- a/Sources/CoreCommands/Options.swift +++ b/Sources/CoreCommands/Options.swift @@ -579,83 +579,36 @@ public struct TestLibraryOptions: ParsableArguments { help: "Enable support for XCTest") public var explicitlyEnableXCTestSupport: Bool? - /// Whether to enable support for XCTest. - public var enableXCTestSupport: Bool { - // Default to enabled. - explicitlyEnableXCTestSupport ?? true - } - - /// Whether to enable support for swift-testing (as explicitly specified by the user.) + /// Whether to enable support for Swift Testing (as explicitly specified by the user.) /// - /// Callers (other than `swift package init`) will generally want to use - /// ``enableSwiftTestingLibrarySupport(swiftCommandState:)`` since it will - /// take into account whether the package has a dependency on swift-testing. - @Flag(name: .customLong("experimental-swift-testing"), + /// Callers will generally want to use ``enableSwiftTestingLibrarySupport`` since it will + /// have the correct default value if the user didn't specify one. + @Flag(name: .customLong("swift-testing"), inversion: .prefixedEnableDisable, - help: "Enable experimental support for swift-testing") + help: "Enable support for Swift Testing") public var explicitlyEnableSwiftTestingLibrarySupport: Bool? - /// Whether to enable support for swift-testing. - public func enableSwiftTestingLibrarySupport( - swiftCommandState: SwiftCommandState - ) throws -> Bool { - // Honor the user's explicit command-line selection, if any. - if let callerSuppliedValue = explicitlyEnableSwiftTestingLibrarySupport { - return callerSuppliedValue - } - - // If the active package has a dependency on swift-testing, automatically enable support for it so that extra steps are not needed. - let workspace = try swiftCommandState.getActiveWorkspace() - let root = try swiftCommandState.getWorkspaceRoot() - let rootManifests = try temp_await { - workspace.loadRootManifests( - packages: root.packages, - observabilityScope: swiftCommandState.observabilityScope, - completion: $0 - ) - } - - // Is swift-testing among the dependencies of the package being built? - // If so, enable support. - let isEnabledByDependency = rootManifests.values.lazy - .flatMap(\.dependencies) - .map(\.identity) - .map(String.init(describing:)) - .contains("swift-testing") - if isEnabledByDependency { - swiftCommandState.observabilityScope.emit(debug: "Enabling swift-testing support due to its presence as a package dependency.") - return true - } - - // Is swift-testing the package being built itself (unlikely)? If so, - // enable support. - let isEnabledByName = root.packages.lazy - .map(PackageIdentity.init(path:)) - .map(String.init(describing:)) - .contains("swift-testing") - if isEnabledByName { - swiftCommandState.observabilityScope.emit(debug: "Enabling swift-testing support because it is a root package.") - return true + /// Legacy experimental equivalent of ``explicitlyEnableSwiftTestingLibrarySupport``. + /// + /// This option will be removed in a future update. + @Flag(name: .customLong("experimental-swift-testing"), + inversion: .prefixedEnableDisable, + help: .private) + public var explicitlyEnableExperimentalSwiftTestingLibrarySupport: Bool? + + /// Test whether or not a given library is enabled. + public func isEnabled(_ library: BuildParameters.Testing.Library) -> Bool { + switch library { + case .xctest: + explicitlyEnableXCTestSupport ?? true + case .swiftTesting: + explicitlyEnableSwiftTestingLibrarySupport ?? explicitlyEnableExperimentalSwiftTestingLibrarySupport ?? true } - - // Default to disabled since swift-testing is experimental (opt-in.) - return false } - /// Get the set of enabled testing libraries. - public func enabledTestingLibraries( - swiftCommandState: SwiftCommandState - ) throws -> Set { - var result = Set() - - if enableXCTestSupport { - result.insert(.xctest) - } - if try enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { - result.insert(.swiftTesting) - } - - return result + /// The list of enabled testing libraries. + public var enabledTestingLibraries: [BuildParameters.Testing.Library] { + [.xctest, .swiftTesting].filter(isEnabled) } } diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index 351321eecd8..2540f4384fa 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -387,6 +387,24 @@ public final class UserToolchain: Toolchain { ) } +#if os(macOS) + public func getSwiftTestingHelper() throws -> AbsolutePath { + // The helper would be located in `.build/` directory when + // SwiftPM is built locally and `usr/libexec/swift/pm` directory in + // an installed version. + let binDirectories = self.swiftSDK.toolset.rootPaths + + self.swiftSDK.toolset.rootPaths.map { + $0.parentDirectory.appending(components: ["libexec", "swift", "pm"]) + } + + return try UserToolchain.getTool( + "swiftpm-testing-helper", + binDirectories: binDirectories, + fileSystem: self.fileSystem + ) + } +#endif + internal static func deriveSwiftCFlags( triple: Triple, swiftSDK: SwiftSDK, diff --git a/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift b/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift index bdfb66bb6b8..697a24b65ff 100644 --- a/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift +++ b/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift @@ -38,17 +38,6 @@ extension BuildParameters { explicitlySpecifiedPath: AbsolutePath? ) - /// Whether this test product style requires additional, derived test targets, i.e. there must be additional test targets, beyond those - /// listed explicitly in the package manifest, created in order to add additional behavior (such as entry point logic). - public var requiresAdditionalDerivedTestTargets: Bool { - switch self { - case .loadableBundle: - return false - case .entryPointExecutable: - return true - } - } - /// The explicitly-specified entry point file path, if this style of test product supports it and a path was specified. public var explicitlySpecifiedEntryPointPath: AbsolutePath? { switch self { @@ -113,9 +102,6 @@ extension BuildParameters { } } - /// Which testing library to use for this build. - public var library: Library - public init( configuration: BuildConfiguration, targetTriple: Triple, @@ -123,8 +109,7 @@ extension BuildParameters { enableTestability: Bool? = nil, experimentalTestOutput: Bool = false, forceTestDiscovery: Bool = false, - testEntryPointPath: AbsolutePath? = nil, - library: Library = .xctest + testEntryPointPath: AbsolutePath? = nil ) { self.enableCodeCoverage = enableCodeCoverage self.experimentalTestOutput = experimentalTestOutput @@ -136,11 +121,10 @@ extension BuildParameters { // when building and testing in release mode, one can use the '--disable-testable-imports' flag // to disable testability in `swift test`, but that requires that the tests do not use the testable imports feature self.enableTestability = enableTestability ?? (.debug == configuration) - self.testProductStyle = (targetTriple.isDarwin() && library == .xctest) ? .loadableBundle : .entryPointExecutable( + self.testProductStyle = targetTriple.isDarwin() ? .loadableBundle : .entryPointExecutable( explicitlyEnabledDiscovery: forceTestDiscovery, explicitlySpecifiedPath: testEntryPointPath ) - self.library = library } } } diff --git a/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift b/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift index 5db1b5d6f9f..abade1ddf7d 100644 --- a/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift +++ b/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift @@ -293,16 +293,11 @@ public struct BuildParameters: Encodable { guard !self.triple.isWasm else { return try RelativePath(validating: "\(product.name).wasm") } - switch testingParameters.library { - case .xctest: - let base = "\(product.name).xctest" - if self.triple.isDarwin() { - return try RelativePath(validating: "\(base)/Contents/MacOS/\(product.name)") - } else { - return try RelativePath(validating: base) - } - case .swiftTesting: - return try RelativePath(validating: "\(product.name).swift-testing") + let base = "\(product.name).xctest" + if self.triple.isDarwin() { + return try RelativePath(validating: "\(base)/Contents/MacOS/\(product.name)") + } else { + return try RelativePath(validating: base) } case .macro: #if BUILD_MACROS_AS_DYLIBS diff --git a/Sources/SPMBuildCore/BuiltTestProduct.swift b/Sources/SPMBuildCore/BuiltTestProduct.swift index 881ade7175f..70f31901e3b 100644 --- a/Sources/SPMBuildCore/BuiltTestProduct.swift +++ b/Sources/SPMBuildCore/BuiltTestProduct.swift @@ -28,15 +28,8 @@ public struct BuiltTestProduct: Codable { /// When the test product is not bundled (for instance, when using XCTest on /// non-Darwin targets), this path is equal to ``binaryPath``. public var bundlePath: AbsolutePath { - // Go up the folder hierarchy until we find the .xctest or - // .swift-testing bundle. - let pathExtension: String - switch library { - case .xctest: - pathExtension = ".xctest" - case .swiftTesting: - pathExtension = ".swift-testing" - } + // Go up the folder hierarchy until we find the .xctest bundle. + let pathExtension = ".xctest" let hierarchySequence = sequence(first: binaryPath, next: { $0.isRoot ? nil : $0.parentDirectory }) guard let bundlePath = hierarchySequence.first(where: { $0.basename.hasSuffix(pathExtension) }) else { fatalError("could not find test bundle path from '\(binaryPath)'") @@ -45,18 +38,14 @@ public struct BuiltTestProduct: Codable { return bundlePath } - /// The library used to build this test product. - public var library: BuildParameters.Testing.Library - /// Creates a new instance. /// - Parameters: /// - productName: The test product name. /// - binaryPath: The path of the test binary. /// - packagePath: The path to the package this product was declared in. - public init(productName: String, binaryPath: AbsolutePath, packagePath: AbsolutePath, library: BuildParameters.Testing.Library) { + public init(productName: String, binaryPath: AbsolutePath, packagePath: AbsolutePath) { self.productName = productName self.binaryPath = binaryPath self.packagePath = packagePath - self.library = library } } diff --git a/Sources/Workspace/InitPackage.swift b/Sources/Workspace/InitPackage.swift index 78e3d783be2..866fb292835 100644 --- a/Sources/Workspace/InitPackage.swift +++ b/Sources/Workspace/InitPackage.swift @@ -276,7 +276,7 @@ public final class InitPackage { dependencies.append(#".package(url: "https://github.com/swiftlang/swift-syntax.git", from: "\#(self.installedSwiftPMConfiguration.swiftSyntaxVersionForMacroTemplate.description)")"#) } if options.supportedTestingLibraries.contains(.swiftTesting) { - dependencies.append(#".package(url: "https://github.com/swiftlang/swift-testing.git", from: "0.2.0")"#) + dependencies.append(#".package(url: "https://github.com/apple/swift-testing.git", from: "0.11.0")"#) } if !dependencies.isEmpty { let dependencies = dependencies.map { dependency in diff --git a/Sources/XCBuildSupport/XcodeBuildSystem.swift b/Sources/XCBuildSupport/XcodeBuildSystem.swift index 5b8803a0481..bee340282e8 100644 --- a/Sources/XCBuildSupport/XcodeBuildSystem.swift +++ b/Sources/XCBuildSupport/XcodeBuildSystem.swift @@ -58,8 +58,7 @@ public final class XcodeBuildSystem: SPMBuildCore.BuildSystem { BuiltTestProduct( productName: product.name, binaryPath: binaryPath, - packagePath: package.path, - library: buildParameters.testingParameters.library + packagePath: package.path ) ) } diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 6ed336fcbcb..05fe87b93a4 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -1144,17 +1144,16 @@ final class BuildPlanTests: XCTestCase { )) XCTAssertEqual(Set(result.productMap.keys.map(\.productName)), ["APackageTests"]) - #if os(macOS) - XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), ["ATarget", "BTarget", "ATargetTests"]) - #else - XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), [ + var expectedTargets: Set = [ "APackageTests", - "APackageDiscoveredTests", "ATarget", "ATargetTests", "BTarget", - ]) - #endif + ] +#if !os(macOS) + expectedTargets.insert("APackageDiscoveredTests") +#endif + XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), expectedTargets) } func testBasicReleasePackage() throws { @@ -2211,13 +2210,7 @@ final class BuildPlanTests: XCTestCase { observabilityScope: observability.topScope )) result.checkProductsCount(1) - #if os(macOS) - result.checkTargetsCount(2) - #else - // On non-Apple platforms, when a custom entry point file is present (e.g. XCTMain.swift), there is one - // additional target for the synthesized test entry point. result.checkTargetsCount(3) - #endif let buildPath = result.plan.productsBuildPath @@ -2286,6 +2279,8 @@ final class BuildPlanTests: XCTestCase { buildPath.appending(components: "Modules", "Foo.swiftmodule").pathString, "-Xlinker", "-add_ast_path", "-Xlinker", buildPath.appending(components: "Modules", "FooTests.swiftmodule").pathString, + "-Xlinker", "-add_ast_path", "-Xlinker", + buildPath.appending(components: "Modules", "PkgPackageTests.swiftmodule").pathString, "-g", ] ) diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 4fde584289c..41f91562ef8 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -197,7 +197,6 @@ final class TestCommandTests: CommandsTestCase { XCTAssertNoMatch(stdout, .contains("testExample2")) XCTAssertNoMatch(stdout, .contains("testExample3")) XCTAssertNoMatch(stdout, .contains("testExample4")) - XCTAssertMatch(stderr, .contains("No matching test cases were run")) } } diff --git a/Tests/WorkspaceTests/InitTests.swift b/Tests/WorkspaceTests/InitTests.swift index b2f46e2f2f7..96958148cfc 100644 --- a/Tests/WorkspaceTests/InitTests.swift +++ b/Tests/WorkspaceTests/InitTests.swift @@ -179,7 +179,7 @@ final class InitTests: XCTestCase { XCTAssertMatch(manifestContents, .contains(#".tvOS(.v13)"#)) XCTAssertMatch(manifestContents, .contains(#".watchOS(.v6)"#)) XCTAssertMatch(manifestContents, .contains(#".macCatalyst(.v13)"#)) - XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.2.0""#)) + XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.11.0""#)) XCTAssertMatch(manifestContents, .contains(#".product(name: "Testing", package: "swift-testing")"#)) let testFile = path.appending("Tests").appending("FooTests").appending("FooTests.swift") @@ -222,7 +222,7 @@ final class InitTests: XCTestCase { XCTAssertMatch(manifestContents, .contains(#".tvOS(.v13)"#)) XCTAssertMatch(manifestContents, .contains(#".watchOS(.v6)"#)) XCTAssertMatch(manifestContents, .contains(#".macCatalyst(.v13)"#)) - XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.2.0""#)) + XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.11.0""#)) XCTAssertMatch(manifestContents, .contains(#".product(name: "Testing", package: "swift-testing")"#)) let testFile = path.appending("Tests").appending("FooTests").appending("FooTests.swift") @@ -262,7 +262,7 @@ final class InitTests: XCTestCase { let manifest = path.appending("Package.swift") XCTAssertFileExists(manifest) let manifestContents: String = try localFileSystem.readFileContents(manifest) - XCTAssertNoMatch(manifestContents, .contains(#"swift-testing.git", from: "0.2.0""#)) + XCTAssertNoMatch(manifestContents, .contains(#"swift-testing.git", from: "0.11.0""#)) XCTAssertNoMatch(manifestContents, .contains(#".product(name: "Testing", package: "swift-testing")"#)) XCTAssertNoMatch(manifestContents, .contains(#".testTarget"#))