diff --git a/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift b/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift index 7a0c293105d..a8adc3923e6 100644 --- a/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift +++ b/Sources/Build/BuildManifest/LLBuildManifestBuilder.swift @@ -112,7 +112,9 @@ public class LLBuildManifestBuilder { } } - try self.addTestDiscoveryGenerationCommand() + if self.plan.destinationBuildParameters.testingParameters.library == .xctest { + try self.addTestDiscoveryGenerationCommand() + } try self.addTestEntryPointGenerationCommand() // Create command for all products in the plan. @@ -308,7 +310,9 @@ extension LLBuildManifestBuilder { let outputs = testEntryPointTarget.target.sources.paths - let mainFileName = TestEntryPointTool.mainFileName + let mainFileName = TestEntryPointTool.mainFileName( + for: self.plan.destinationBuildParameters.testingParameters.library + ) guard let mainOutput = (outputs.first { $0.basename == mainFileName }) else { throw InternalError("main output (\(mainFileName)) not found") } diff --git a/Sources/Build/BuildPlan/BuildPlan+Product.swift b/Sources/Build/BuildPlan/BuildPlan+Product.swift index 9782103e689..3967aef58f8 100644 --- a/Sources/Build/BuildPlan/BuildPlan+Product.swift +++ b/Sources/Build/BuildPlan/BuildPlan+Product.swift @@ -275,8 +275,10 @@ extension BuildPlan { } // Add derived test targets, if necessary - if product.type == .test, let derivedTestTargets = derivedTestTargetsMap[product.id] { - staticTargets.append(contentsOf: derivedTestTargets) + if buildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets { + if product.type == .test, let derivedTestTargets = derivedTestTargetsMap[product.id] { + staticTargets.append(contentsOf: derivedTestTargets) + } } return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths, providedLibraries, availableTools) diff --git a/Sources/Build/BuildPlan/BuildPlan+Test.swift b/Sources/Build/BuildPlan/BuildPlan+Test.swift index 96b20532d8e..53cce82bc97 100644 --- a/Sources/Build/BuildPlan/BuildPlan+Test.swift +++ b/Sources/Build/BuildPlan/BuildPlan+Test.swift @@ -34,12 +34,13 @@ extension BuildPlan { _ fileSystem: FileSystem, _ observabilityScope: ObservabilityScope ) throws -> [(product: ResolvedProduct, discoveryTargetBuildDescription: SwiftModuleBuildDescription?, entryPointTargetBuildDescription: SwiftModuleBuildDescription)] { - var explicitlyEnabledDiscovery = false - var explicitlySpecifiedPath: AbsolutePath? - if case let .entryPointExecutable(caseExplicitlyEnabledDiscovery, caseExplicitlySpecifiedPath) = destinationBuildParameters.testingParameters.testProductStyle { - explicitlyEnabledDiscovery = caseExplicitlyEnabledDiscovery - explicitlySpecifiedPath = caseExplicitlySpecifiedPath + guard destinationBuildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets, + case .entryPointExecutable(let explicitlyEnabledDiscovery, let explicitlySpecifiedPath) = + destinationBuildParameters.testingParameters.testProductStyle + else { + throw InternalError("makeTestManifestTargets should not be used for build plan which does not require additional derived test targets") } + let isEntryPointPathSpecifiedExplicitly = explicitlySpecifiedPath != nil var isDiscoveryEnabledRedundantly = explicitlyEnabledDiscovery && !isEntryPointPathSpecifiedExplicitly @@ -115,7 +116,7 @@ extension BuildPlan { resolvedTargetDependencies: [ResolvedModule.Dependency] ) throws -> SwiftModuleBuildDescription { let entryPointDerivedDir = destinationBuildParameters.buildPath.appending(components: "\(testProduct.name).derived") - let entryPointMainFileName = TestEntryPointTool.mainFileName + let entryPointMainFileName = TestEntryPointTool.mainFileName(for: destinationBuildParameters.testingParameters.library) let entryPointMainFile = entryPointDerivedDir.appending(component: entryPointMainFileName) let entryPointSources = Sources(paths: [entryPointMainFile], root: entryPointDerivedDir) @@ -152,17 +153,18 @@ extension BuildPlan { let swiftTargetDependencies: [Module.Dependency] let resolvedTargetDependencies: [ResolvedModule.Dependency] - if destinationBuildParameters.triple.isDarwin() { - discoveryTargets = nil - swiftTargetDependencies = [] - resolvedTargetDependencies = [] - } else { + switch destinationBuildParameters.testingParameters.library { + case .xctest: discoveryTargets = try generateDiscoveryTargets() swiftTargetDependencies = [.module(discoveryTargets!.target, conditions: [])] resolvedTargetDependencies = [.module(discoveryTargets!.resolved, conditions: [])] + case .swiftTesting: + discoveryTargets = nil + swiftTargetDependencies = testProduct.modules.map { .module($0.underlying, conditions: []) } + resolvedTargetDependencies = testProduct.modules.map { .module($0, conditions: []) } } - if !destinationBuildParameters.triple.isDarwin(), let entryPointResolvedTarget = testProduct.testEntryPointModule { + if let entryPointResolvedTarget = testProduct.testEntryPointModule { if isEntryPointPathSpecifiedExplicitly || explicitlyEnabledDiscovery { if isEntryPointPathSpecifiedExplicitly { // Allow using the explicitly-specified test entry point target, but still perform test discovery and thus declare a dependency on the discovery modules. diff --git a/Sources/Build/BuildPlan/BuildPlan.swift b/Sources/Build/BuildPlan/BuildPlan.swift index d49842fdfea..0d89793d520 100644 --- a/Sources/Build/BuildPlan/BuildPlan.swift +++ b/Sources/Build/BuildPlan/BuildPlan.swift @@ -473,29 +473,31 @@ public class BuildPlan: SPMBuildCore.BuildPlan { } // Plan the derived test targets, if necessary. - let derivedTestTargets = try Self.makeDerivedTestTargets( - testProducts: productMap.values.filter { - $0.product.type == .test - }, - destinationBuildParameters: destinationBuildParameters, - toolsBuildParameters: toolsBuildParameters, - shouldDisableSandbox: self.shouldDisableSandbox, - self.fileSystem, - self.observabilityScope - ) - for item in derivedTestTargets { - var derivedTestTargets = [item.entryPointTargetBuildDescription.target] - - targetMap[item.entryPointTargetBuildDescription.target.id] = .swift( - item.entryPointTargetBuildDescription + if destinationBuildParameters.testingParameters.testProductStyle.requiresAdditionalDerivedTestTargets { + let derivedTestTargets = try Self.makeDerivedTestTargets( + testProducts: productMap.values.filter { + $0.product.type == .test + }, + destinationBuildParameters: destinationBuildParameters, + toolsBuildParameters: toolsBuildParameters, + shouldDisableSandbox: self.shouldDisableSandbox, + self.fileSystem, + self.observabilityScope ) + for item in derivedTestTargets { + var derivedTestTargets = [item.entryPointTargetBuildDescription.target] - if let discoveryTargetBuildDescription = item.discoveryTargetBuildDescription { - targetMap[discoveryTargetBuildDescription.target.id] = .swift(discoveryTargetBuildDescription) - derivedTestTargets.append(discoveryTargetBuildDescription.target) - } + targetMap[item.entryPointTargetBuildDescription.target.id] = .swift( + item.entryPointTargetBuildDescription + ) + + if let discoveryTargetBuildDescription = item.discoveryTargetBuildDescription { + targetMap[discoveryTargetBuildDescription.target.id] = .swift(discoveryTargetBuildDescription) + derivedTestTargets.append(discoveryTargetBuildDescription.target) + } - self.derivedTestTargetsMap[item.product.id] = derivedTestTargets + self.derivedTestTargetsMap[item.product.id] = derivedTestTargets + } } self.buildToolPluginInvocationResults = buildToolPluginInvocationResults diff --git a/Sources/Build/LLBuildCommands.swift b/Sources/Build/LLBuildCommands.swift index f083947608b..a414119c236 100644 --- a/Sources/Build/LLBuildCommands.swift +++ b/Sources/Build/LLBuildCommands.swift @@ -50,8 +50,8 @@ extension IndexStore.TestCaseClass.TestMethod { } extension TestEntryPointTool { - public static var mainFileName: String { - "runner.swift" + public static func mainFileName(for library: BuildParameters.Testing.Library) -> String { + "runner-\(library).swift" } } @@ -105,76 +105,74 @@ final class TestDiscoveryCommand: CustomLLBuildCommand, TestBuildCommand { private func execute(fileSystem: Basics.FileSystem, tool: TestDiscoveryTool) throws { let outputs = tool.outputs.compactMap { try? AbsolutePath(validating: $0.name) } - if case .loadableBundle = context.productsBuildParameters.testingParameters.testProductStyle { - // When building an XCTest bundle, test discovery is handled by the - // test harness process (i.e. this is the Darwin path.) + switch self.context.productsBuildParameters.testingParameters.library { + case .swiftTesting: for file in outputs { try fileSystem.writeIfChanged(path: file, string: "") } - return - } - - let index = self.context.productsBuildParameters.indexStore - let api = try self.context.indexStoreAPI.get() - let store = try IndexStore.open(store: TSCAbsolutePath(index), api: api) - - // FIXME: We can speed this up by having one llbuild command per object file. - let tests = try store - .listTests(in: tool.inputs.map { try TSCAbsolutePath(AbsolutePath(validating: $0.name)) }) - - let testsByModule = Dictionary(grouping: tests, by: { $0.module.spm_mangledToC99ExtendedIdentifier() }) - - // Find the main file path. - guard let mainFile = outputs.first(where: { path in - path.basename == TestDiscoveryTool.mainFileName - }) else { - throw InternalError("main output (\(TestDiscoveryTool.mainFileName)) not found") - } + case .xctest: + let index = self.context.productsBuildParameters.indexStore + let api = try self.context.indexStoreAPI.get() + let store = try IndexStore.open(store: TSCAbsolutePath(index), api: api) + + // FIXME: We can speed this up by having one llbuild command per object file. + let tests = try store + .listTests(in: tool.inputs.map { try TSCAbsolutePath(AbsolutePath(validating: $0.name)) }) + + let testsByModule = Dictionary(grouping: tests, by: { $0.module.spm_mangledToC99ExtendedIdentifier() }) + + // Find the main file path. + guard let mainFile = outputs.first(where: { path in + path.basename == TestDiscoveryTool.mainFileName + }) else { + throw InternalError("main output (\(TestDiscoveryTool.mainFileName)) not found") + } - // Write one file for each test module. - // - // We could write everything in one file but that can easily run into type conflicts due - // in complex packages with large number of test modules. - for file in outputs where file != mainFile { - // FIXME: This is relying on implementation detail of the output but passing the - // the context all the way through is not worth it right now. - let module = file.basenameWithoutExt.spm_mangledToC99ExtendedIdentifier() - - guard let tests = testsByModule[module] else { - // This module has no tests so just write an empty file for it. - try fileSystem.writeFileContents(file, bytes: "") - continue + // Write one file for each test module. + // + // We could write everything in one file but that can easily run into type conflicts due + // in complex packages with large number of test modules. + for file in outputs where file != mainFile { + // FIXME: This is relying on implementation detail of the output but passing the + // the context all the way through is not worth it right now. + let module = file.basenameWithoutExt.spm_mangledToC99ExtendedIdentifier() + + guard let tests = testsByModule[module] else { + // This module has no tests so just write an empty file for it. + try fileSystem.writeFileContents(file, bytes: "") + continue + } + try write( + tests: tests, + forModule: module, + fileSystem: fileSystem, + path: file + ) } - try write( - tests: tests, - forModule: module, - fileSystem: fileSystem, - path: file - ) - } - let testsKeyword = tests.isEmpty ? "let" : "var" + let testsKeyword = tests.isEmpty ? "let" : "var" - // Write the main file. - let stream = try LocalFileOutputByteStream(mainFile) + // Write the main file. + let stream = try LocalFileOutputByteStream(mainFile) - stream.send( - #""" - import XCTest + stream.send( + #""" + import XCTest - @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") - @MainActor - public func __allDiscoveredTests() -> [XCTestCaseEntry] { - \#(testsKeyword) tests = [XCTestCaseEntry]() + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") + @MainActor + public func __allDiscoveredTests() -> [XCTestCaseEntry] { + \#(testsKeyword) tests = [XCTestCaseEntry]() - \#(testsByModule.keys.map { "tests += __\($0)__allTests()" }.joined(separator: "\n ")) + \#(testsByModule.keys.map { "tests += __\($0)__allTests()" }.joined(separator: "\n ")) - return tests - } - """# - ) + return tests + } + """# + ) - stream.flush() + stream.flush() + } } override func execute( @@ -203,7 +201,9 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { let outputs = tool.outputs.compactMap { try? AbsolutePath(validating: $0.name) } // Find the main output file - let mainFileName = TestEntryPointTool.mainFileName + let mainFileName = TestEntryPointTool.mainFileName( + for: self.context.productsBuildParameters.testingParameters.library + ) guard let mainFile = outputs.first(where: { path in path.basename == mainFileName }) else { @@ -213,100 +213,62 @@ final class TestEntryPointCommand: CustomLLBuildCommand, TestBuildCommand { // Write the main file. let stream = try LocalFileOutputByteStream(mainFile) - // Find the inputs, which are the names of the test discovery module(s) - let inputs = tool.inputs.compactMap { try? AbsolutePath(validating: $0.name) } - let discoveryModuleNames = inputs.map(\.basenameWithoutExt) - - let testObservabilitySetup: String - let buildParameters = self.context.productsBuildParameters - if buildParameters.testingParameters.experimentalTestOutput && buildParameters.triple.supportsTestSummary { - testObservabilitySetup = "_ = SwiftPMXCTestObserver()\n" - } else { - testObservabilitySetup = "" - } - - let isXCTMainAvailable: String = switch buildParameters.testingParameters.testProductStyle { - case .entryPointExecutable: - "canImport(XCTest)" - case .loadableBundle: - "false" - } - - /// On WASI, we can't block the main thread, so XCTestMain is defined as async. - let awaitXCTMainKeyword = if buildParameters.triple.isWASI() { - "await" - } else { - "" - } - - var needsAsyncMainWorkaround = false - if buildParameters.triple.isLinux() { - // FIXME: work around crash on Amazon Linux 2 when main function is async (rdar://128303921) - needsAsyncMainWorkaround = true - } else if buildParameters.triple.isDarwin() { -#if compiler(<5.10) - // FIXME: work around duplicate async_Main symbols (SEE https://github.com/swiftlang/swift/pull/69113) - needsAsyncMainWorkaround = true -#endif - } - - stream.send( - #""" - #if canImport(Testing) - import Testing - #endif - - #if \#(isXCTMainAvailable) - \#(generateTestObservationCode(buildParameters: buildParameters)) - - import XCTest - \#(discoveryModuleNames.map { "import \($0)" }.joined(separator: "\n")) - #endif + switch self.context.productsBuildParameters.testingParameters.library { + case .swiftTesting: + stream.send( + #""" + #if canImport(Testing) + import Testing + #endif - @main - @available(macOS 10.15, iOS 11, watchOS 4, tvOS 11, *) - @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") - struct Runner { - private static func testingLibrary() -> String { - var iterator = CommandLine.arguments.makeIterator() - while let argument = iterator.next() { - if argument == "--testing-library", let libraryName = iterator.next() { - return libraryName.lowercased() - } + @main struct Runner { + static func main() async { + #if canImport(Testing) + await Testing.__swiftPMEntryPoint() as Never + #endif } - - // Fallback if not specified: run XCTest (legacy behavior) - return "xctest" } + """# + ) + case .xctest: + // Find the inputs, which are the names of the test discovery module(s) + let inputs = tool.inputs.compactMap { try? AbsolutePath(validating: $0.name) } + let discoveryModuleNames = inputs.map(\.basenameWithoutExt) + + let testObservabilitySetup: String + let buildParameters = self.context.productsBuildParameters + if buildParameters.testingParameters.experimentalTestOutput && buildParameters.triple.supportsTestSummary { + testObservabilitySetup = "_ = SwiftPMXCTestObserver()\n" + } else { + testObservabilitySetup = "" + } - #if \#(needsAsyncMainWorkaround) - @_silgen_name("$ss13_runAsyncMainyyyyYaKcF") - private static func _runAsyncMain(_ asyncFun: @Sendable @escaping () async throws -> ()) - #endif + stream.send( + #""" + \#(generateTestObservationCode(buildParameters: buildParameters)) - static func main() \#(needsAsyncMainWorkaround ? "" : "async") { - let testingLibrary = Self.testingLibrary() - #if canImport(Testing) - if testingLibrary == "swift-testing" { - #if \#(needsAsyncMainWorkaround) - _runAsyncMain { - await Testing.__swiftPMEntryPoint() as Never - } - #else - await Testing.__swiftPMEntryPoint() as Never - #endif + import XCTest + \#(discoveryModuleNames.map { "import \($0)" }.joined(separator: "\n")) + + @main + @available(*, deprecated, message: "Not actually deprecated. Marked as deprecated to allow inclusion of deprecated tests (which test deprecated functionality) without warnings") + struct Runner { + #if os(WASI) + /// On WASI, we can't block the main thread, so XCTestMain is defined as async. + static func main() async { + \#(testObservabilitySetup) + await XCTMain(__allDiscoveredTests()) as Never } - #endif - #if \#(isXCTMainAvailable) - if testingLibrary == "xctest" { + #else + static func main() { \#(testObservabilitySetup) - \#(awaitXCTMainKeyword) XCTMain(__allDiscoveredTests()) as Never + XCTMain(__allDiscoveredTests()) as Never } #endif } - } - """# - ) + """# + ) + } stream.flush() } diff --git a/Sources/Build/LLBuildDescription.swift b/Sources/Build/LLBuildDescription.swift index b8426d7cb7b..1576aade5be 100644 --- a/Sources/Build/LLBuildDescription.swift +++ b/Sources/Build/LLBuildDescription.swift @@ -140,7 +140,8 @@ public struct BuildDescription: Codable { try BuiltTestProduct( productName: desc.product.name, binaryPath: desc.binaryPath, - packagePath: desc.package.path + packagePath: desc.package.path, + library: desc.buildParameters.testingParameters.library ) } self.pluginDescriptions = pluginDescriptions diff --git a/Sources/Commands/PackageCommands/Init.swift b/Sources/Commands/PackageCommands/Init.swift index 43903533508..6263e84b835 100644 --- a/Sources/Commands/PackageCommands/Init.swift +++ b/Sources/Commands/PackageCommands/Init.swift @@ -54,23 +54,20 @@ extension SwiftPackageCommand { throw InternalError("Could not find the current working directory") } - let packageName = self.packageName ?? cwd.basename - - // Which testing libraries should be used? XCTest is on by default, - // but Swift Testing must remain off by default until it is present - // in the Swift toolchain. - var supportedTestingLibraries = Set() - if testLibraryOptions.isEnabled(.xctest) { - supportedTestingLibraries.insert(.xctest) + // NOTE: Do not use testLibraryOptions.enabledTestingLibraries(swiftCommandState:) here + // because the package doesn't exist yet, so there are no dependencies for it to query. + var testingLibraries: Set = [] + if testLibraryOptions.enableXCTestSupport { + testingLibraries.insert(.xctest) } - if testLibraryOptions.explicitlyEnableSwiftTestingLibrarySupport == true || testLibraryOptions.explicitlyEnableExperimentalSwiftTestingLibrarySupport == true { - supportedTestingLibraries.insert(.swiftTesting) + if testLibraryOptions.explicitlyEnableSwiftTestingLibrarySupport == true { + testingLibraries.insert(.swiftTesting) } - + let packageName = self.packageName ?? cwd.basename let initPackage = try InitPackage( name: packageName, packageType: initMode, - supportedTestingLibraries: supportedTestingLibraries, + supportedTestingLibraries: testingLibraries, destinationPath: cwd, installedSwiftPMConfiguration: swiftCommandState.getHostToolchain().installedSwiftPMConfiguration, fileSystem: swiftCommandState.fileSystem diff --git a/Sources/Commands/SwiftBuildCommand.swift b/Sources/Commands/SwiftBuildCommand.swift index 3f8f6299706..dbbeadec15b 100644 --- a/Sources/Commands/SwiftBuildCommand.swift +++ b/Sources/Commands/SwiftBuildCommand.swift @@ -151,12 +151,35 @@ public struct SwiftBuildCommand: AsyncSwiftCommand { var productsBuildParameters = try swiftCommandState.productsBuildParameters var toolsBuildParameters = try swiftCommandState.toolsBuildParameters + // Clean out the code coverage directory that may contain stale + // profraw files from a previous run of the code coverage tool. if self.options.enableCodeCoverage { + try swiftCommandState.fileSystem.removeFileTree(swiftCommandState.productsBuildParameters.codeCovPath) productsBuildParameters.testingParameters.enableCodeCoverage = true toolsBuildParameters.testingParameters.enableCodeCoverage = true } - try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) + if case .allIncludingTests = subset { + func updateTestingParameters(of buildParameters: inout BuildParameters, library: BuildParameters.Testing.Library) { + buildParameters.testingParameters = .init( + configuration: buildParameters.configuration, + targetTriple: buildParameters.triple, + enableCodeCoverage: buildParameters.testingParameters.enableCodeCoverage, + enableTestability: buildParameters.testingParameters.enableTestability, + experimentalTestOutput: buildParameters.testingParameters.experimentalTestOutput, + forceTestDiscovery: globalOptions.build.enableTestDiscovery, + testEntryPointPath: globalOptions.build.testEntryPointPath, + library: library + ) + } + for library in try options.testLibraryOptions.enabledTestingLibraries(swiftCommandState: swiftCommandState) { + updateTestingParameters(of: &productsBuildParameters, library: library) + updateTestingParameters(of: &toolsBuildParameters, library: library) + try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) + } + } else { + try build(swiftCommandState, subset: subset, productsBuildParameters: productsBuildParameters, toolsBuildParameters: toolsBuildParameters) + } } private func build( diff --git a/Sources/Commands/SwiftTestCommand.swift b/Sources/Commands/SwiftTestCommand.swift index 00edd3a30ab..1fdee4d2239 100644 --- a/Sources/Commands/SwiftTestCommand.swift +++ b/Sources/Commands/SwiftTestCommand.swift @@ -247,116 +247,102 @@ public struct SwiftTestCommand: AsyncSwiftCommand { @OptionGroup() var options: TestCommandOptions - private func run(_ swiftCommandState: SwiftCommandState, buildParameters: BuildParameters, testProducts: [BuiltTestProduct]) async throws { + // MARK: - XCTest + + private func xctestRun(_ swiftCommandState: SwiftCommandState) async throws { + // validate XCTest available on darwin based systems + let toolchain = try swiftCommandState.getTargetToolchain() + if case let .unsupported(reason) = try swiftCommandState.getHostToolchain().swiftSDK.xctestSupport { + if let reason { + throw TestError.xctestNotAvailable(reason: reason) + } else { + throw TestError.xcodeNotInstalled + } + } else if toolchain.targetTriple.isDarwin() && toolchain.xctestPath == nil { + throw TestError.xcodeNotInstalled + } + + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: .xctest) + // Remove test output from prior runs and validate priors. - if self.options.enableExperimentalTestOutput && buildParameters.triple.supportsTestSummary { - _ = try? localFileSystem.removeFileTree(buildParameters.testOutputPath) + if self.options.enableExperimentalTestOutput && productsBuildParameters.triple.supportsTestSummary { + _ = try? localFileSystem.removeFileTree(productsBuildParameters.testOutputPath) } - var results = [TestRunner.Result]() + let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState, library: .xctest) + if !self.options.shouldRunInParallel { + let xctestArgs = try xctestArgs(for: testProducts, swiftCommandState: swiftCommandState) + try await runTestProducts( + testProducts, + additionalArguments: xctestArgs, + productsBuildParameters: productsBuildParameters, + swiftCommandState: swiftCommandState, + library: .xctest + ) + } else { + let testSuites = try TestingSupport.getTestSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + enableCodeCoverage: options.enableCodeCoverage, + shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, + experimentalTestOutput: options.enableExperimentalTestOutput, + sanitizers: globalOptions.build.sanitizers + ) + let tests = try testSuites + .filteredTests(specifier: options.testCaseSpecifier) + .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) - // Run XCTest. - if options.testLibraryOptions.isEnabled(.xctest) { - // validate XCTest available on darwin based systems - let toolchain = try swiftCommandState.getTargetToolchain() - if case let .unsupported(reason) = try swiftCommandState.getHostToolchain().swiftSDK.xctestSupport { - if let reason { - throw TestError.xctestNotAvailable(reason: reason) - } else { - throw TestError.xcodeNotInstalled - } - } else if toolchain.targetTriple.isDarwin() && toolchain.xctestPath == nil { - throw TestError.xcodeNotInstalled + // If there were no matches, emit a warning and exit. + if tests.isEmpty { + swiftCommandState.observabilityScope.emit(.noMatchingTests) + try generateXUnitOutputIfRequested(for: [], swiftCommandState: swiftCommandState) + return } - if !self.options.shouldRunInParallel { - let (xctestArgs, testCount) = try xctestArgs(for: testProducts, swiftCommandState: swiftCommandState) - let result = try await runTestProducts( - testProducts, - additionalArguments: xctestArgs, - productsBuildParameters: buildParameters, - swiftCommandState: swiftCommandState, - library: .xctest - ) - if result == .success && testCount == 0 { - results.append(.noMatchingTests) - } else { - results.append(result) - } - } else { - let testSuites = try TestingSupport.getTestSuites( - in: testProducts, - swiftCommandState: swiftCommandState, - enableCodeCoverage: options.enableCodeCoverage, - shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, - experimentalTestOutput: options.enableExperimentalTestOutput, - sanitizers: globalOptions.build.sanitizers - ) - let tests = try testSuites - .filteredTests(specifier: options.testCaseSpecifier) - .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) - - let result: TestRunner.Result - let testResults: [ParallelTestRunner.TestResult] - if tests.isEmpty { - testResults = [] - result = .noMatchingTests - } else { - // Run the tests using the parallel runner. - let runner = ParallelTestRunner( - bundlePaths: testProducts.map { $0.bundlePath }, - cancellator: swiftCommandState.cancellator, - toolchain: toolchain, - numJobs: options.numberOfWorkers ?? ProcessInfo.processInfo.activeProcessorCount, - buildOptions: globalOptions.build, - productsBuildParameters: buildParameters, - shouldOutputSuccess: swiftCommandState.logLevel <= .info, - observabilityScope: swiftCommandState.observabilityScope - ) + // Clean out the code coverage directory that may contain stale + // profraw files from a previous run of the code coverage tool. + if self.options.enableCodeCoverage { + try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) + } - testResults = try runner.run(tests) - result = runner.ranSuccessfully ? .success : .failure - } + // Run the tests using the parallel runner. + let runner = ParallelTestRunner( + bundlePaths: testProducts.map { $0.bundlePath }, + cancellator: swiftCommandState.cancellator, + toolchain: toolchain, + numJobs: options.numberOfWorkers ?? ProcessInfo.processInfo.activeProcessorCount, + buildOptions: globalOptions.build, + productsBuildParameters: productsBuildParameters, + shouldOutputSuccess: swiftCommandState.logLevel <= .info, + observabilityScope: swiftCommandState.observabilityScope + ) + + let testResults = try runner.run(tests) - try generateXUnitOutputIfRequested(for: testResults, swiftCommandState: swiftCommandState) - results.append(result) + try generateXUnitOutputIfRequested(for: testResults, swiftCommandState: swiftCommandState) + + // process code Coverage if request + if self.options.enableCodeCoverage, runner.ranSuccessfully { + try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState, library: .xctest) } - } - // Run Swift Testing (parallel or not, it has a single entry point.) - if options.testLibraryOptions.isEnabled(.swiftTesting) { - results.append( - try await runTestProducts( - testProducts, - additionalArguments: [], - productsBuildParameters: buildParameters, - swiftCommandState: swiftCommandState, - library: .swiftTesting - ) - ) - } + if !runner.ranSuccessfully { + swiftCommandState.executionStatus = .failure + } - switch results.reduce() { - case .success: - // Nothing to do here. - break - case .failure: - swiftCommandState.executionStatus = .failure - if self.options.enableExperimentalTestOutput { - try Self.handleTestOutput(productsBuildParameters: buildParameters, packagePath: testProducts[0].packagePath) + if self.options.enableExperimentalTestOutput, !runner.ranSuccessfully { + try Self.handleTestOutput(productsBuildParameters: productsBuildParameters, packagePath: testProducts[0].packagePath) } - case .noMatchingTests: - swiftCommandState.observabilityScope.emit(.noMatchingTests) } } - private func xctestArgs(for testProducts: [BuiltTestProduct], swiftCommandState: SwiftCommandState) throws -> (arguments: [String], testCount: Int) { + private func xctestArgs(for testProducts: [BuiltTestProduct], swiftCommandState: SwiftCommandState) throws -> [String] { switch options.testCaseSpecifier { case .none: if case .skip = options.skippedTests(fileSystem: swiftCommandState.fileSystem) { fallthrough } else { - return ([], 0) + return [] } case .regex, .specific, .skip: @@ -378,7 +364,12 @@ public struct SwiftTestCommand: AsyncSwiftCommand { .filteredTests(specifier: options.testCaseSpecifier) .skippedTests(specifier: options.skippedTests(fileSystem: swiftCommandState.fileSystem)) - return (TestRunner.xctestArguments(forTestSpecifiers: tests.map(\.specifier)), tests.count) + // If there were no matches, emit a warning. + if tests.isEmpty { + swiftCommandState.observabilityScope.emit(.noMatchingTests) + } + + return TestRunner.xctestArguments(forTestSpecifiers: tests.map(\.specifier)) } } @@ -398,6 +389,21 @@ public struct SwiftTestCommand: AsyncSwiftCommand { try generator.generate(at: xUnitOutput) } + // MARK: - swift-testing + + private func swiftTestingRun(_ swiftCommandState: SwiftCommandState) async throws { + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: .swiftTesting) + let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState, library: .swiftTesting) + let additionalArguments = Array(CommandLine.arguments.dropFirst()) + try await runTestProducts( + testProducts, + additionalArguments: additionalArguments, + productsBuildParameters: productsBuildParameters, + swiftCommandState: swiftCommandState, + library: .swiftTesting + ) + } + // MARK: - Common implementation public func run(_ swiftCommandState: SwiftCommandState) async throws { @@ -416,22 +422,12 @@ public struct SwiftTestCommand: AsyncSwiftCommand { let command = try List.parse() try command.run(swiftCommandState) } else { - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) - let testProducts = try buildTestsIfNeeded(swiftCommandState: swiftCommandState) - - // Clean out the code coverage directory that may contain stale - // profraw files from a previous run of the code coverage tool. - if self.options.enableCodeCoverage { - try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) + if try options.testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { + try await swiftTestingRun(swiftCommandState) } - - try await run(swiftCommandState, buildParameters: productsBuildParameters, testProducts: testProducts) - - // process code Coverage if request - if self.options.enableCodeCoverage, swiftCommandState.executionStatus != .failure { - try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState) + if options.testLibraryOptions.enableXCTestSupport { + try await xctestRun(swiftCommandState) } - } } @@ -441,11 +437,11 @@ public struct SwiftTestCommand: AsyncSwiftCommand { productsBuildParameters: BuildParameters, swiftCommandState: SwiftCommandState, library: BuildParameters.Testing.Library - ) async throws -> TestRunner.Result { - // Pass through all arguments from the command line to Swift Testing. - var additionalArguments = additionalArguments - if library == .swiftTesting { - additionalArguments += CommandLine.arguments.dropFirst() + ) async throws { + // Clean out the code coverage directory that may contain stale + // profraw files from a previous run of the code coverage tool. + if self.options.enableCodeCoverage { + try swiftCommandState.fileSystem.removeFileTree(productsBuildParameters.codeCovPath) } let toolchain = try swiftCommandState.getTargetToolchain() @@ -456,15 +452,8 @@ public struct SwiftTestCommand: AsyncSwiftCommand { library: library ) - let runnerPaths: [AbsolutePath] = switch library { - case .xctest: - testProducts.map(\.bundlePath) - case .swiftTesting: - testProducts.map(\.binaryPath) - } - let runner = TestRunner( - bundlePaths: runnerPaths, + bundlePaths: testProducts.map { library == .xctest ? $0.bundlePath : $0.binaryPath }, additionalArguments: additionalArguments, cancellator: swiftCommandState.cancellator, toolchain: toolchain, @@ -474,11 +463,22 @@ public struct SwiftTestCommand: AsyncSwiftCommand { ) // Finally, run the tests. - return runner.test(outputHandler: { + let ranSuccessfully = runner.test(outputHandler: { // command's result output goes on stdout // ie "swift test" should output to stdout print($0, terminator: "") }) + if !ranSuccessfully { + swiftCommandState.executionStatus = .failure + } + + if self.options.enableCodeCoverage, ranSuccessfully { + try await processCodeCoverage(testProducts, swiftCommandState: swiftCommandState, library: library) + } + + if self.options.enableExperimentalTestOutput, !ranSuccessfully { + try Self.handleTestOutput(productsBuildParameters: productsBuildParameters, packagePath: testProducts[0].packagePath) + } } private static func handleTestOutput(productsBuildParameters: BuildParameters, packagePath: AbsolutePath) throws { @@ -515,7 +515,8 @@ public struct SwiftTestCommand: AsyncSwiftCommand { /// Processes the code coverage data and emits a json. private func processCodeCoverage( _ testProducts: [BuiltTestProduct], - swiftCommandState: SwiftCommandState + swiftCommandState: SwiftCommandState, + library: BuildParameters.Testing.Library ) async throws { let workspace = try swiftCommandState.getActiveWorkspace() let root = try swiftCommandState.getWorkspaceRoot() @@ -528,23 +529,23 @@ public struct SwiftTestCommand: AsyncSwiftCommand { } // Merge all the profraw files to produce a single profdata file. - try mergeCodeCovRawDataFiles(swiftCommandState: swiftCommandState) + try mergeCodeCovRawDataFiles(swiftCommandState: swiftCommandState, library: library) - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) for product in testProducts { // Export the codecov data as JSON. let jsonPath = productsBuildParameters.codeCovAsJSONPath(packageName: rootManifest.displayName) - try exportCodeCovAsJSON(to: jsonPath, testBinary: product.binaryPath, swiftCommandState: swiftCommandState) + try exportCodeCovAsJSON(to: jsonPath, testBinary: product.binaryPath, swiftCommandState: swiftCommandState, library: library) } } /// Merges all profraw profiles in codecoverage directory into default.profdata file. - private func mergeCodeCovRawDataFiles(swiftCommandState: SwiftCommandState) throws { + private func mergeCodeCovRawDataFiles(swiftCommandState: SwiftCommandState, library: BuildParameters.Testing.Library) throws { // Get the llvm-prof tool. let llvmProf = try swiftCommandState.getTargetToolchain().getLLVMProf() // Get the profraw files. - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) let codeCovFiles = try swiftCommandState.fileSystem.getDirectoryContents(productsBuildParameters.codeCovPath) // Construct arguments for invoking the llvm-prof tool. @@ -564,11 +565,12 @@ public struct SwiftTestCommand: AsyncSwiftCommand { private func exportCodeCovAsJSON( to path: AbsolutePath, testBinary: AbsolutePath, - swiftCommandState: SwiftCommandState + swiftCommandState: SwiftCommandState, + library: BuildParameters.Testing.Library ) throws { // Export using the llvm-cov tool. let llvmCov = try swiftCommandState.getTargetToolchain().getLLVMCov() - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) let args = [ llvmCov.pathString, "export", @@ -588,9 +590,10 @@ public struct SwiftTestCommand: AsyncSwiftCommand { /// /// - Returns: The paths to the build test products. private func buildTestsIfNeeded( - swiftCommandState: SwiftCommandState + swiftCommandState: SwiftCommandState, + library: BuildParameters.Testing.Library ) throws -> [BuiltTestProduct] { - let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest(options: self.options) + let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest(options: self.options, library: library) return try Commands.buildTestsIfNeeded( swiftCommandState: swiftCommandState, productsBuildParameters: productsBuildParameters, @@ -618,7 +621,7 @@ public struct SwiftTestCommand: AsyncSwiftCommand { throw StringError("'--num-workers' must be greater than zero") } - guard options.testLibraryOptions.isEnabled(.xctest) else { + if !options.testLibraryOptions.enableXCTestSupport { throw StringError("'--num-workers' is only supported when testing with XCTest") } } @@ -645,7 +648,7 @@ extension SwiftTestCommand { guard let rootManifest = rootManifests.values.first else { throw StringError("invalid manifests at \(root.packages)") } - let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(enableCodeCoverage: true) + let (productsBuildParameters, _) = try swiftCommandState.buildParametersForTest(enableCodeCoverage: true, library: .xctest) print(productsBuildParameters.codeCovAsJSONPath(packageName: rootManifest.displayName)) } } @@ -689,10 +692,41 @@ extension SwiftTestCommand { @Flag(name: [.customLong("list-tests"), .customShort("l")], help: .hidden) var _deprecated_passthrough: Bool = false - func run(_ swiftCommandState: SwiftCommandState) throws { + // MARK: - XCTest + + private func xctestRun(_ swiftCommandState: SwiftCommandState) throws { + let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest( + enableCodeCoverage: false, + shouldSkipBuilding: sharedOptions.shouldSkipBuilding, + library: .xctest + ) + let testProducts = try buildTestsIfNeeded( + swiftCommandState: swiftCommandState, + productsBuildParameters: productsBuildParameters, + toolsBuildParameters: toolsBuildParameters + ) + let testSuites = try TestingSupport.getTestSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + enableCodeCoverage: false, + shouldSkipBuilding: sharedOptions.shouldSkipBuilding, + experimentalTestOutput: false, + sanitizers: globalOptions.build.sanitizers + ) + + // Print the tests. + for test in testSuites.allTests { + print(test.specifier) + } + } + + // MARK: - swift-testing + + private func swiftTestingRun(_ swiftCommandState: SwiftCommandState) throws { let (productsBuildParameters, toolsBuildParameters) = try swiftCommandState.buildParametersForTest( enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding + shouldSkipBuilding: sharedOptions.shouldSkipBuilding, + library: .swiftTesting ) let testProducts = try buildTestsIfNeeded( swiftCommandState: swiftCommandState, @@ -708,43 +742,36 @@ extension SwiftTestCommand { library: .swiftTesting ) - if testLibraryOptions.isEnabled(.xctest) { - let testSuites = try TestingSupport.getTestSuites( - in: testProducts, - swiftCommandState: swiftCommandState, - enableCodeCoverage: false, - shouldSkipBuilding: sharedOptions.shouldSkipBuilding, - experimentalTestOutput: false, - sanitizers: globalOptions.build.sanitizers - ) - - // Print the tests. - for test in testSuites.allTests { - print(test.specifier) - } + let additionalArguments = ["--list-tests"] + CommandLine.arguments.dropFirst() + let runner = TestRunner( + bundlePaths: testProducts.map(\.binaryPath), + additionalArguments: additionalArguments, + cancellator: swiftCommandState.cancellator, + toolchain: toolchain, + testEnv: testEnv, + observabilityScope: swiftCommandState.observabilityScope, + library: .swiftTesting + ) + + // Finally, run the tests. + let ranSuccessfully = runner.test(outputHandler: { + // command's result output goes on stdout + // ie "swift test" should output to stdout + print($0, terminator: "") + }) + if !ranSuccessfully { + swiftCommandState.executionStatus = .failure } + } - if testLibraryOptions.isEnabled(.swiftTesting) { - let additionalArguments = ["--list-tests"] + CommandLine.arguments.dropFirst() - let runner = TestRunner( - bundlePaths: testProducts.map(\.binaryPath), - additionalArguments: additionalArguments, - cancellator: swiftCommandState.cancellator, - toolchain: toolchain, - testEnv: testEnv, - observabilityScope: swiftCommandState.observabilityScope, - library: .swiftTesting - ) - - // Finally, run the tests. - let result = runner.test(outputHandler: { - // command's result output goes on stdout - // ie "swift test" should output to stdout - print($0, terminator: "") - }) - if result == .failure { - swiftCommandState.executionStatus = .failure - } + // MARK: - Common implementation + + func run(_ swiftCommandState: SwiftCommandState) throws { + if try testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { + try swiftTestingRun(swiftCommandState) + } + if testLibraryOptions.enableXCTestSupport { + try xctestRun(swiftCommandState) } } @@ -844,6 +871,12 @@ final class TestRunner { self.library = library } + /// Executes and returns execution status. Prints test output on standard streams if requested + /// - Returns: Boolean indicating if test execution returned code 0, and the output stream result + func test(outputHandler: @escaping (String) -> Void) -> Bool { + (test(outputHandler: outputHandler) as Result) != .failure + } + /// The result of running the test(s). enum Result: Equatable { /// The test(s) ran successfully. @@ -860,41 +893,39 @@ final class TestRunner { /// Executes and returns execution status. Prints test output on standard streams if requested /// - Returns: Result of spawning and running the test process, and the output stream result + @_disfavoredOverload func test(outputHandler: @escaping (String) -> Void) -> Result { var results = [Result]() for path in self.bundlePaths { let testSuccess = self.test(at: path, outputHandler: outputHandler) results.append(testSuccess) } - return results.reduce() + if results.contains(.failure) { + return .failure + } else if results.isEmpty || results.contains(.success) { + return .success + } else { + return .noMatchingTests + } } /// Constructs arguments to execute XCTest. private func args(forTestAt testPath: AbsolutePath) throws -> [String] { var args: [String] = [] -#if os(macOS) - switch library { - case .xctest: + #if os(macOS) + if library == .xctest { guard let xctestPath = self.toolchain.xctestPath else { throw TestError.xcodeNotInstalled } - args += [xctestPath.pathString] - case .swiftTesting: - let helper = try self.toolchain.getSwiftTestingHelper() - args += [helper.pathString, "--test-bundle-path", testPath.pathString] + args = [xctestPath.pathString] + args += additionalArguments + args += [testPath.pathString] + return args } - args += additionalArguments - args += [testPath.pathString] -#else - args += [testPath.pathString] - args += additionalArguments -#endif + #endif - if library == .swiftTesting { - // HACK: tell the test bundle/executable that we want to run Swift Testing, not XCTest. - // XCTest doesn't understand this argument (yet), so don't pass it there. - args += ["--testing-library", "swift-testing"] - } + args += [testPath.description] + args += additionalArguments return args } @@ -939,19 +970,6 @@ final class TestRunner { } } -extension Collection where Element == TestRunner.Result { - /// Reduce all results in this collection into a single result. - func reduce() -> Element { - if contains(.failure) { - return .failure - } else if isEmpty || contains(.success) { - return .success - } else { - return .noMatchingTests - } - } -} - /// A class to run tests in parallel. final class ParallelTestRunner { /// An enum representing result of a unit test execution. @@ -1083,20 +1101,20 @@ final class ParallelTestRunner { toolchain: self.toolchain, testEnv: testEnv, observabilityScope: self.observabilityScope, - library: .xctest // swift-testing does not use ParallelTestRunner + library: .xctest ) var output = "" let outputLock = NSLock() let start = DispatchTime.now() - let result = testRunner.test(outputHandler: { _output in outputLock.withLock{ output += _output }}) + let success = testRunner.test(outputHandler: { _output in outputLock.withLock{ output += _output }}) let duration = start.distance(to: .now()) - if result == .failure { + if !success { self.ranSuccessfully = false } self.finishedTests.enqueue(TestResult( unitTest: test, output: output, - success: result != .failure, + success: success, duration: duration )) } @@ -1347,14 +1365,21 @@ final class XUnitGenerator { extension SwiftCommandState { func buildParametersForTest( - options: TestCommandOptions + options: TestCommandOptions, + library: BuildParameters.Testing.Library ) throws -> (productsBuildParameters: BuildParameters, toolsBuildParameters: BuildParameters) { - try self.buildParametersForTest( + var result = try self.buildParametersForTest( enableCodeCoverage: options.enableCodeCoverage, enableTestability: options.enableTestableImports, shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, - experimentalTestOutput: options.enableExperimentalTestOutput + experimentalTestOutput: options.enableExperimentalTestOutput, + library: library ) + if try options.testLibraryOptions.enableSwiftTestingLibrarySupport(swiftCommandState: self) { + result.productsBuildParameters.flags.swiftCompilerFlags += ["-DSWIFT_PM_SUPPORTS_SWIFT_TESTING"] + result.toolsBuildParameters.flags.swiftCompilerFlags += ["-DSWIFT_PM_SUPPORTS_SWIFT_TESTING"] + } + return result } } diff --git a/Sources/Commands/Utilities/PluginDelegate.swift b/Sources/Commands/Utilities/PluginDelegate.swift index cfcf2aa2ec6..c9e1fbd69a9 100644 --- a/Sources/Commands/Utilities/PluginDelegate.swift +++ b/Sources/Commands/Utilities/PluginDelegate.swift @@ -288,11 +288,11 @@ final class PluginDelegate: PluginInvocationDelegate { // Run the test — for now we run the sequentially so we can capture accurate timing results. let startTime = DispatchTime.now() - let result = testRunner.test(outputHandler: { _ in }) // this drops the tests output + let success = testRunner.test(outputHandler: { _ in }) // this drops the tests output let duration = Double(startTime.distance(to: .now()).milliseconds() ?? 0) / 1000.0 - numFailedTests += (result != .failure) ? 0 : 1 + numFailedTests += success ? 0 : 1 testResults.append( - .init(name: testName, result: (result != .failure) ? .succeeded : .failed, duration: duration) + .init(name: testName, result: success ? .succeeded : .failed, duration: duration) ) } diff --git a/Sources/Commands/Utilities/TestingSupport.swift b/Sources/Commands/Utilities/TestingSupport.swift index a29acbde1ae..1e49d6defec 100644 --- a/Sources/Commands/Utilities/TestingSupport.swift +++ b/Sources/Commands/Utilities/TestingSupport.swift @@ -118,7 +118,8 @@ enum TestingSupport { destinationBuildParameters: swiftCommandState.buildParametersForTest( enableCodeCoverage: enableCodeCoverage, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput + experimentalTestOutput: experimentalTestOutput, + library: .xctest ).productsBuildParameters, sanitizers: sanitizers, library: .xctest @@ -133,7 +134,8 @@ enum TestingSupport { toolchain: try swiftCommandState.getTargetToolchain(), destinationBuildParameters: swiftCommandState.buildParametersForTest( enableCodeCoverage: enableCodeCoverage, - shouldSkipBuilding: shouldSkipBuilding + shouldSkipBuilding: shouldSkipBuilding, + library: .xctest ).productsBuildParameters, sanitizers: sanitizers, library: .xctest @@ -162,6 +164,10 @@ enum TestingSupport { env["NO_COLOR"] = "1" } + // Set an environment variable to indicate which library's test product + // is being executed. + env["SWIFT_PM_TEST_LIBRARY"] = String(describing: library) + // Add the code coverage related variables. if buildParameters.testingParameters.enableCodeCoverage { // Defines the path at which the profraw files will be written on test execution. @@ -171,7 +177,7 @@ enum TestingSupport { // execution but is required when the tests are running in parallel as // SwiftPM repeatedly invokes the test binary with the test case name as // the filter. - let codecovProfile = buildParameters.buildPath.appending(components: "codecov", "\(library)%m.profraw") + let codecovProfile = buildParameters.buildPath.appending(components: "codecov", "default%m.profraw") env["LLVM_PROFILE_FILE"] = codecovProfile.pathString } #if !os(macOS) @@ -189,11 +195,6 @@ enum TestingSupport { env.appendPath(key: "DYLD_LIBRARY_PATH", value: sdkPlatformFrameworksPath.lib.pathString) } - // We aren't using XCTest's harness logic to run Swift Testing tests. - if library == .xctest { - env["SWIFT_TESTING_ENABLED"] = "0" - } - // Fast path when no sanitizers are enabled. if sanitizers.isEmpty { return env @@ -220,21 +221,24 @@ extension SwiftCommandState { enableCodeCoverage: Bool, enableTestability: Bool? = nil, shouldSkipBuilding: Bool = false, - experimentalTestOutput: Bool = false + experimentalTestOutput: Bool = false, + library: BuildParameters.Testing.Library ) throws -> (productsBuildParameters: BuildParameters, toolsBuildParameters: BuildParameters) { let productsBuildParameters = buildParametersForTest( modifying: try productsBuildParameters, enableCodeCoverage: enableCodeCoverage, enableTestability: enableTestability, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput + experimentalTestOutput: experimentalTestOutput, + library: library ) let toolsBuildParameters = buildParametersForTest( modifying: try toolsBuildParameters, enableCodeCoverage: enableCodeCoverage, enableTestability: enableTestability, shouldSkipBuilding: shouldSkipBuilding, - experimentalTestOutput: experimentalTestOutput + experimentalTestOutput: experimentalTestOutput, + library: library ) return (productsBuildParameters, toolsBuildParameters) } @@ -244,7 +248,8 @@ extension SwiftCommandState { enableCodeCoverage: Bool, enableTestability: Bool?, shouldSkipBuilding: Bool, - experimentalTestOutput: Bool + experimentalTestOutput: Bool, + library: BuildParameters.Testing.Library ) -> BuildParameters { var parameters = parameters @@ -261,7 +266,8 @@ extension SwiftCommandState { configuration: parameters.configuration, targetTriple: parameters.triple, forceTestDiscovery: explicitlyEnabledDiscovery, - testEntryPointPath: explicitlySpecifiedPath + testEntryPointPath: explicitlySpecifiedPath, + library: library ) parameters.testingParameters.enableCodeCoverage = enableCodeCoverage diff --git a/Sources/CoreCommands/Options.swift b/Sources/CoreCommands/Options.swift index bcb2a4a3566..4e6aab11080 100644 --- a/Sources/CoreCommands/Options.swift +++ b/Sources/CoreCommands/Options.swift @@ -580,36 +580,83 @@ public struct TestLibraryOptions: ParsableArguments { help: "Enable support for XCTest") public var explicitlyEnableXCTestSupport: Bool? - /// Whether to enable support for Swift Testing (as explicitly specified by the user.) - /// - /// Callers will generally want to use ``enableSwiftTestingLibrarySupport`` since it will - /// have the correct default value if the user didn't specify one. - @Flag(name: .customLong("swift-testing"), - inversion: .prefixedEnableDisable, - help: "Enable support for Swift Testing") - public var explicitlyEnableSwiftTestingLibrarySupport: Bool? + /// Whether to enable support for XCTest. + public var enableXCTestSupport: Bool { + // Default to enabled. + explicitlyEnableXCTestSupport ?? true + } - /// Legacy experimental equivalent of ``explicitlyEnableSwiftTestingLibrarySupport``. + /// Whether to enable support for swift-testing (as explicitly specified by the user.) /// - /// This option will be removed in a future update. + /// Callers (other than `swift package init`) will generally want to use + /// ``enableSwiftTestingLibrarySupport(swiftCommandState:)`` since it will + /// take into account whether the package has a dependency on swift-testing. @Flag(name: .customLong("experimental-swift-testing"), inversion: .prefixedEnableDisable, - help: .private) - public var explicitlyEnableExperimentalSwiftTestingLibrarySupport: Bool? - - /// Test whether or not a given library is enabled. - public func isEnabled(_ library: BuildParameters.Testing.Library) -> Bool { - switch library { - case .xctest: - explicitlyEnableXCTestSupport ?? true - case .swiftTesting: - explicitlyEnableSwiftTestingLibrarySupport ?? explicitlyEnableExperimentalSwiftTestingLibrarySupport ?? true + help: "Enable experimental support for swift-testing") + public var explicitlyEnableSwiftTestingLibrarySupport: Bool? + + /// Whether to enable support for swift-testing. + public func enableSwiftTestingLibrarySupport( + swiftCommandState: SwiftCommandState + ) throws -> Bool { + // Honor the user's explicit command-line selection, if any. + if let callerSuppliedValue = explicitlyEnableSwiftTestingLibrarySupport { + return callerSuppliedValue + } + + // If the active package has a dependency on swift-testing, automatically enable support for it so that extra steps are not needed. + let workspace = try swiftCommandState.getActiveWorkspace() + let root = try swiftCommandState.getWorkspaceRoot() + let rootManifests = try temp_await { + workspace.loadRootManifests( + packages: root.packages, + observabilityScope: swiftCommandState.observabilityScope, + completion: $0 + ) } + + // Is swift-testing among the dependencies of the package being built? + // If so, enable support. + let isEnabledByDependency = rootManifests.values.lazy + .flatMap(\.dependencies) + .map(\.identity) + .map(String.init(describing:)) + .contains("swift-testing") + if isEnabledByDependency { + swiftCommandState.observabilityScope.emit(debug: "Enabling swift-testing support due to its presence as a package dependency.") + return true + } + + // Is swift-testing the package being built itself (unlikely)? If so, + // enable support. + let isEnabledByName = root.packages.lazy + .map(PackageIdentity.init(path:)) + .map(String.init(describing:)) + .contains("swift-testing") + if isEnabledByName { + swiftCommandState.observabilityScope.emit(debug: "Enabling swift-testing support because it is a root package.") + return true + } + + // Default to disabled since swift-testing is experimental (opt-in.) + return false } - /// The list of enabled testing libraries. - public var enabledTestingLibraries: [BuildParameters.Testing.Library] { - [.xctest, .swiftTesting].filter(isEnabled) + /// Get the set of enabled testing libraries. + public func enabledTestingLibraries( + swiftCommandState: SwiftCommandState + ) throws -> Set { + var result = Set() + + if enableXCTestSupport { + result.insert(.xctest) + } + if try enableSwiftTestingLibrarySupport(swiftCommandState: swiftCommandState) { + result.insert(.swiftTesting) + } + + return result } } diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index acb18db35df..10a0e995f05 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -387,24 +387,6 @@ public final class UserToolchain: Toolchain { ) } -#if os(macOS) - public func getSwiftTestingHelper() throws -> AbsolutePath { - // The helper would be located in `.build/` directory when - // SwiftPM is built locally and `usr/libexec/swift/pm` directory in - // an installed version. - let binDirectories = self.swiftSDK.toolset.rootPaths + - self.swiftSDK.toolset.rootPaths.map { - $0.parentDirectory.appending(components: ["libexec", "swift", "pm"]) - } - - return try UserToolchain.getTool( - "swiftpm-testing-helper", - binDirectories: binDirectories, - fileSystem: self.fileSystem - ) - } -#endif - internal static func deriveSwiftCFlags( triple: Triple, swiftSDK: SwiftSDK, diff --git a/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift b/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift index 697a24b65ff..bdfb66bb6b8 100644 --- a/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift +++ b/Sources/SPMBuildCore/BuildParameters/BuildParameters+Testing.swift @@ -38,6 +38,17 @@ extension BuildParameters { explicitlySpecifiedPath: AbsolutePath? ) + /// Whether this test product style requires additional, derived test targets, i.e. there must be additional test targets, beyond those + /// listed explicitly in the package manifest, created in order to add additional behavior (such as entry point logic). + public var requiresAdditionalDerivedTestTargets: Bool { + switch self { + case .loadableBundle: + return false + case .entryPointExecutable: + return true + } + } + /// The explicitly-specified entry point file path, if this style of test product supports it and a path was specified. public var explicitlySpecifiedEntryPointPath: AbsolutePath? { switch self { @@ -102,6 +113,9 @@ extension BuildParameters { } } + /// Which testing library to use for this build. + public var library: Library + public init( configuration: BuildConfiguration, targetTriple: Triple, @@ -109,7 +123,8 @@ extension BuildParameters { enableTestability: Bool? = nil, experimentalTestOutput: Bool = false, forceTestDiscovery: Bool = false, - testEntryPointPath: AbsolutePath? = nil + testEntryPointPath: AbsolutePath? = nil, + library: Library = .xctest ) { self.enableCodeCoverage = enableCodeCoverage self.experimentalTestOutput = experimentalTestOutput @@ -121,10 +136,11 @@ extension BuildParameters { // when building and testing in release mode, one can use the '--disable-testable-imports' flag // to disable testability in `swift test`, but that requires that the tests do not use the testable imports feature self.enableTestability = enableTestability ?? (.debug == configuration) - self.testProductStyle = targetTriple.isDarwin() ? .loadableBundle : .entryPointExecutable( + self.testProductStyle = (targetTriple.isDarwin() && library == .xctest) ? .loadableBundle : .entryPointExecutable( explicitlyEnabledDiscovery: forceTestDiscovery, explicitlySpecifiedPath: testEntryPointPath ) + self.library = library } } } diff --git a/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift b/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift index abade1ddf7d..5db1b5d6f9f 100644 --- a/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift +++ b/Sources/SPMBuildCore/BuildParameters/BuildParameters.swift @@ -293,11 +293,16 @@ public struct BuildParameters: Encodable { guard !self.triple.isWasm else { return try RelativePath(validating: "\(product.name).wasm") } - let base = "\(product.name).xctest" - if self.triple.isDarwin() { - return try RelativePath(validating: "\(base)/Contents/MacOS/\(product.name)") - } else { - return try RelativePath(validating: base) + switch testingParameters.library { + case .xctest: + let base = "\(product.name).xctest" + if self.triple.isDarwin() { + return try RelativePath(validating: "\(base)/Contents/MacOS/\(product.name)") + } else { + return try RelativePath(validating: base) + } + case .swiftTesting: + return try RelativePath(validating: "\(product.name).swift-testing") } case .macro: #if BUILD_MACROS_AS_DYLIBS diff --git a/Sources/SPMBuildCore/BuiltTestProduct.swift b/Sources/SPMBuildCore/BuiltTestProduct.swift index 70f31901e3b..881ade7175f 100644 --- a/Sources/SPMBuildCore/BuiltTestProduct.swift +++ b/Sources/SPMBuildCore/BuiltTestProduct.swift @@ -28,8 +28,15 @@ public struct BuiltTestProduct: Codable { /// When the test product is not bundled (for instance, when using XCTest on /// non-Darwin targets), this path is equal to ``binaryPath``. public var bundlePath: AbsolutePath { - // Go up the folder hierarchy until we find the .xctest bundle. - let pathExtension = ".xctest" + // Go up the folder hierarchy until we find the .xctest or + // .swift-testing bundle. + let pathExtension: String + switch library { + case .xctest: + pathExtension = ".xctest" + case .swiftTesting: + pathExtension = ".swift-testing" + } let hierarchySequence = sequence(first: binaryPath, next: { $0.isRoot ? nil : $0.parentDirectory }) guard let bundlePath = hierarchySequence.first(where: { $0.basename.hasSuffix(pathExtension) }) else { fatalError("could not find test bundle path from '\(binaryPath)'") @@ -38,14 +45,18 @@ public struct BuiltTestProduct: Codable { return bundlePath } + /// The library used to build this test product. + public var library: BuildParameters.Testing.Library + /// Creates a new instance. /// - Parameters: /// - productName: The test product name. /// - binaryPath: The path of the test binary. /// - packagePath: The path to the package this product was declared in. - public init(productName: String, binaryPath: AbsolutePath, packagePath: AbsolutePath) { + public init(productName: String, binaryPath: AbsolutePath, packagePath: AbsolutePath, library: BuildParameters.Testing.Library) { self.productName = productName self.binaryPath = binaryPath self.packagePath = packagePath + self.library = library } } diff --git a/Sources/Workspace/InitPackage.swift b/Sources/Workspace/InitPackage.swift index 866fb292835..78e3d783be2 100644 --- a/Sources/Workspace/InitPackage.swift +++ b/Sources/Workspace/InitPackage.swift @@ -276,7 +276,7 @@ public final class InitPackage { dependencies.append(#".package(url: "https://github.com/swiftlang/swift-syntax.git", from: "\#(self.installedSwiftPMConfiguration.swiftSyntaxVersionForMacroTemplate.description)")"#) } if options.supportedTestingLibraries.contains(.swiftTesting) { - dependencies.append(#".package(url: "https://github.com/apple/swift-testing.git", from: "0.11.0")"#) + dependencies.append(#".package(url: "https://github.com/swiftlang/swift-testing.git", from: "0.2.0")"#) } if !dependencies.isEmpty { let dependencies = dependencies.map { dependency in diff --git a/Sources/XCBuildSupport/XcodeBuildSystem.swift b/Sources/XCBuildSupport/XcodeBuildSystem.swift index bee340282e8..5b8803a0481 100644 --- a/Sources/XCBuildSupport/XcodeBuildSystem.swift +++ b/Sources/XCBuildSupport/XcodeBuildSystem.swift @@ -58,7 +58,8 @@ public final class XcodeBuildSystem: SPMBuildCore.BuildSystem { BuiltTestProduct( productName: product.name, binaryPath: binaryPath, - packagePath: package.path + packagePath: package.path, + library: buildParameters.testingParameters.library ) ) } diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 587ec472f2a..bd27cfa7e2f 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -1146,16 +1146,17 @@ final class BuildPlanTests: XCTestCase { )) XCTAssertEqual(Set(result.productMap.keys.map(\.productName)), ["APackageTests"]) - var expectedTargets: Set = [ + #if os(macOS) + XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), ["ATarget", "BTarget", "ATargetTests"]) + #else + XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), [ "APackageTests", + "APackageDiscoveredTests", "ATarget", "ATargetTests", "BTarget", - ] -#if !os(macOS) - expectedTargets.insert("APackageDiscoveredTests") -#endif - XCTAssertEqual(Set(result.targetMap.keys.map(\.moduleName)), expectedTargets) + ]) + #endif } func testBasicReleasePackage() throws { @@ -2212,7 +2213,13 @@ final class BuildPlanTests: XCTestCase { observabilityScope: observability.topScope )) result.checkProductsCount(1) + #if os(macOS) + result.checkTargetsCount(2) + #else + // On non-Apple platforms, when a custom entry point file is present (e.g. XCTMain.swift), there is one + // additional target for the synthesized test entry point. result.checkTargetsCount(3) + #endif let buildPath = result.plan.productsBuildPath @@ -2281,8 +2288,6 @@ final class BuildPlanTests: XCTestCase { buildPath.appending(components: "Modules", "Foo.swiftmodule").pathString, "-Xlinker", "-add_ast_path", "-Xlinker", buildPath.appending(components: "Modules", "FooTests.swiftmodule").pathString, - "-Xlinker", "-add_ast_path", "-Xlinker", - buildPath.appending(components: "Modules", "PkgPackageTests.swiftmodule").pathString, "-g", ] ) diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index 4c0ef5a494b..7b693cabc8b 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -197,6 +197,7 @@ final class TestCommandTests: CommandsTestCase { XCTAssertNoMatch(stdout, .contains("testExample2")) XCTAssertNoMatch(stdout, .contains("testExample3")) XCTAssertNoMatch(stdout, .contains("testExample4")) + XCTAssertMatch(stderr, .contains("No matching test cases were run")) } } diff --git a/Tests/WorkspaceTests/InitTests.swift b/Tests/WorkspaceTests/InitTests.swift index d6232b7b304..781a63a68fe 100644 --- a/Tests/WorkspaceTests/InitTests.swift +++ b/Tests/WorkspaceTests/InitTests.swift @@ -179,7 +179,7 @@ final class InitTests: XCTestCase { XCTAssertMatch(manifestContents, .contains(#".tvOS(.v13)"#)) XCTAssertMatch(manifestContents, .contains(#".watchOS(.v6)"#)) XCTAssertMatch(manifestContents, .contains(#".macCatalyst(.v13)"#)) - XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.11.0""#)) + XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.2.0""#)) XCTAssertMatch(manifestContents, .contains(#".product(name: "Testing", package: "swift-testing")"#)) let testFile = path.appending("Tests").appending("FooTests").appending("FooTests.swift") @@ -222,7 +222,7 @@ final class InitTests: XCTestCase { XCTAssertMatch(manifestContents, .contains(#".tvOS(.v13)"#)) XCTAssertMatch(manifestContents, .contains(#".watchOS(.v6)"#)) XCTAssertMatch(manifestContents, .contains(#".macCatalyst(.v13)"#)) - XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.11.0""#)) + XCTAssertMatch(manifestContents, .contains(#"swift-testing.git", from: "0.2.0""#)) XCTAssertMatch(manifestContents, .contains(#".product(name: "Testing", package: "swift-testing")"#)) let testFile = path.appending("Tests").appending("FooTests").appending("FooTests.swift") @@ -262,7 +262,7 @@ final class InitTests: XCTestCase { let manifest = path.appending("Package.swift") XCTAssertFileExists(manifest) let manifestContents: String = try localFileSystem.readFileContents(manifest) - XCTAssertNoMatch(manifestContents, .contains(#"swift-testing.git", from: "0.11.0""#)) + XCTAssertNoMatch(manifestContents, .contains(#"swift-testing.git", from: "0.2.0""#)) XCTAssertNoMatch(manifestContents, .contains(#".product(name: "Testing", package: "swift-testing")"#)) XCTAssertNoMatch(manifestContents, .contains(#".testTarget"#))