diff --git a/.gitignore b/.gitignore index a22e17b2b..748a71bfa 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ default.profraw *.vsix .vscode-test .build +assets/test/**/Package.resolved \ No newline at end of file diff --git a/docs/images/coverage-render.png b/docs/images/coverage-render.png index dc8205411..b39759590 100644 Binary files a/docs/images/coverage-render.png and b/docs/images/coverage-render.png differ diff --git a/docs/images/coverage-report.png b/docs/images/coverage-report.png index 26eba102a..3ca118f12 100644 Binary files a/docs/images/coverage-report.png and b/docs/images/coverage-report.png differ diff --git a/docs/images/coverage-run.png b/docs/images/coverage-run.png index 5bc6b2272..0df2f5f4c 100644 Binary files a/docs/images/coverage-run.png and b/docs/images/coverage-run.png differ diff --git a/docs/test-coverage.md b/docs/test-coverage.md index 820aaa588..c8afa609f 100644 --- a/docs/test-coverage.md +++ b/docs/test-coverage.md @@ -2,17 +2,14 @@ Test coverage is a measurement of how much of your code is tested by your tests. It defines how many lines of code were actually run when you ran your tests and how many were not. When a line of code is not run by your tests it will not have been tested and perhaps you need to extend your tests. -The Swift extension has an option to run your tests and record what code has been hit or missed by your tests. +The Swift extension integrates with VSCode's Code Coverage APIs to record what code has been hit or missed by your tests. ![](images/coverage-run.png) -Once this is run an overview report will be displayed listing all the source files in your project and how many lines were hit by tests, how many lines were missed, how lines of source code there is in total and a percentage of those that were hit. You can click on each file to open that file in Visual Studio Code. If you close the report you can always get it back by running the command `Show Test Coverage Report`. There is also a setting `Coverage: Display Report after Run` to control whether the test coverage report is shown immediately after running tests. +Once you've performed a code coverage run a coverage report will be displayed in a section of the primary side bar. This report lists all the source files in your project and what percentage of lines were hit by tests. You can click on each file to open that file in the code editor. If you close the report you can always get it back by running the command `Test: Open Coverage`. ![](images/coverage-report.png) -If you would like a more detailed view of the results there is a command `Toggle Display of Test Coverage Results` to toggle an in editor view of the coverage results. This will color the background of hit and missed lines of code with different colours. By default a hit line of code gets a green background and a missed line get a red background, although these are editable in the settings. +After generating code coverage lines numbers in covered files will be coloured red or green depending on if they ran during the test run. Hovering over the line numbers shows how many times each line was run. Hitting the "Toggle Inline Coverage" link that appears when hovering over the line numbers will keep this information visible. ![](images/coverage-render.png) - -An additional UI status item is displayed in the status bar at the bottom of the screen showing the hit percentage for the currently open file. This status item can be set to be always visible or only when coverage information is available. If you have it set to be visible all the time it can be used as a button to toggle the display of the in editor coverage results. - diff --git a/package-lock.json b/package-lock.json index 224b7c87b..e4af994de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@types/mocha": "^10.0.1", "@types/node": "^18.19.33", "@types/plist": "^3.0.5", - "@types/vscode": "^1.71.0", + "@types/vscode": "^1.88.0", "@types/xml2js": "^0.4.14", "@typescript-eslint/eslint-plugin": "^7.8.0", "@typescript-eslint/parser": "^7.8.0", @@ -36,7 +36,7 @@ "typescript": "^5.4.5" }, "engines": { - "vscode": "^1.71.0" + "vscode": "^1.88.0" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -855,9 +855,9 @@ "dev": true }, "node_modules/@types/vscode": { - "version": "1.71.0", - "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.71.0.tgz", - "integrity": "sha512-nB50bBC9H/x2CpwW9FzRRRDrTZ7G0/POttJojvN/LiVfzTGfLyQIje1L1QRMdFXK9G41k5UJN/1B9S4of7CSzA==", + "version": "1.89.0", + "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.89.0.tgz", + "integrity": "sha512-TMfGKLSVxfGfoO8JfIE/neZqv7QLwS4nwPwL/NwMvxtAY2230H2I4Z5xx6836pmJvMAzqooRQ4pmLm7RUicP3A==", "dev": true }, "node_modules/@types/xml2js": { @@ -5063,9 +5063,9 @@ "dev": true }, "@types/vscode": { - "version": "1.71.0", - "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.71.0.tgz", - "integrity": "sha512-nB50bBC9H/x2CpwW9FzRRRDrTZ7G0/POttJojvN/LiVfzTGfLyQIje1L1QRMdFXK9G41k5UJN/1B9S4of7CSzA==", + "version": "1.89.0", + "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.89.0.tgz", + "integrity": "sha512-TMfGKLSVxfGfoO8JfIE/neZqv7QLwS4nwPwL/NwMvxtAY2230H2I4Z5xx6836pmJvMAzqooRQ4pmLm7RUicP3A==", "dev": true }, "@types/xml2js": { diff --git a/package.json b/package.json index a9f44f0d2..009c19526 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "url": "https://github.com/swift-server/vscode-swift" }, "engines": { - "vscode": "^1.71.0" + "vscode": "^1.88.0" }, "categories": [ "Programming Languages", @@ -160,16 +160,6 @@ "title": "Insert Function Comment", "category": "Swift" }, - { - "command": "swift.showTestCoverageReport", - "title": "Show Test Coverage Report", - "category": "Swift" - }, - { - "command": "swift.toggleTestCoverage", - "title": "Toggle Display of Test Coverage Results", - "category": "Swift" - }, { "command": "swift.attachDebugger", "title": "Attach to Process...", @@ -453,47 +443,6 @@ } } }, - { - "title": "Test Coverage", - "properties": { - "swift.coverage.displayReportAfterRun": { - "type": "boolean", - "default": true, - "description": "Should test coverage report be shown after running tests", - "order": 1 - }, - "swift.coverage.alwaysShowStatusItem": { - "type": "boolean", - "default": true, - "description": "Always show the test coverage status item. If this is set to true the status item can be clicked on to toggle test coverage display on and off.", - "order": 2 - }, - "swift.coverage.colors.lightMode.hit": { - "type": "string", - "default": "#c0ffc0", - "description": "Light mode theme background color for line of code hit during test coverage.", - "order": 3 - }, - "swift.coverage.colors.lightMode.miss": { - "type": "string", - "default": "#ffc0c0", - "description": "Light mode theme background color for line of code missed during test coverage.", - "order": 4 - }, - "swift.coverage.colors.darkMode.hit": { - "type": "string", - "default": "#003000", - "description": "Dark mode theme background color for line of code hit during test coverage.", - "order": 5 - }, - "swift.coverage.colors.darkMode.miss": { - "type": "string", - "default": "#400000", - "description": "Dark mode theme background color for line of code missed during test coverage.", - "order": 6 - } - } - }, { "title": "Debugger", "properties": { @@ -1177,7 +1126,7 @@ "@types/mocha": "^10.0.1", "@types/node": "^18.19.33", "@types/plist": "^3.0.5", - "@types/vscode": "^1.71.0", + "@types/vscode": "^1.88.0", "@typescript-eslint/eslint-plugin": "^7.8.0", "@typescript-eslint/parser": "^7.8.0", "@vscode/test-electron": "^2.3.8", diff --git a/src/FolderContext.ts b/src/FolderContext.ts index a189bf656..b19cf72ef 100644 --- a/src/FolderContext.ts +++ b/src/FolderContext.ts @@ -21,7 +21,6 @@ import { TestExplorer } from "./TestExplorer/TestExplorer"; import { WorkspaceContext, FolderEvent } from "./WorkspaceContext"; import { BackgroundCompilation } from "./BackgroundCompilation"; import { TaskQueue } from "./tasks/TaskQueue"; -import { LcovResults } from "./coverage/LcovResults"; import { isPathInsidePath } from "./utilities/utilities"; export class FolderContext implements vscode.Disposable { @@ -30,7 +29,6 @@ export class FolderContext implements vscode.Disposable { public hasResolveErrors = false; public testExplorer?: TestExplorer; public taskQueue: TaskQueue; - public lcovResults: LcovResults; /** * FolderContext constructor @@ -49,7 +47,6 @@ export class FolderContext implements vscode.Disposable { this.packageWatcher.install(); this.backgroundCompilation = new BackgroundCompilation(this); this.taskQueue = new TaskQueue(this); - this.lcovResults = new LcovResults(this); } /** dispose of any thing FolderContext holds */ @@ -57,7 +54,6 @@ export class FolderContext implements vscode.Disposable { this.linuxMain?.dispose(); this.packageWatcher.dispose(); this.testExplorer?.dispose(); - this.lcovResults.dispose(); } /** diff --git a/src/TestExplorer/TestParsers/XCTestOutputParser.ts b/src/TestExplorer/TestParsers/XCTestOutputParser.ts index 7e295b5a7..52edc08bb 100644 --- a/src/TestExplorer/TestParsers/XCTestOutputParser.ts +++ b/src/TestExplorer/TestParsers/XCTestOutputParser.ts @@ -14,6 +14,7 @@ import { ITestRunState } from "./TestRunState"; import { sourceLocationToVSCodeLocation } from "../../utilities/utilities"; +import { MarkdownString, Location } from "vscode"; /** Regex for parsing XCTest output */ interface TestRegex { @@ -67,7 +68,69 @@ export const nonDarwinTestRegex = { failedSuite: /^Test Suite '(.*)' failed/, }; -export class XCTestOutputParser { +export interface IXCTestOutputParser { + parseResult(output: string, runState: ITestRunState): void; +} + +export class ParallelXCTestOutputParser implements IXCTestOutputParser { + private outputParser: XCTestOutputParser; + + /** + * Create an ParallelXCTestOutputParser. + * Optional regex can be supplied for tests. + */ + constructor( + private hasMultiLineParallelTestOutput: boolean, + regex?: TestRegex + ) { + this.outputParser = new XCTestOutputParser(regex); + } + + public parseResult(output: string, runState: ITestRunState) { + // From 5.7 to 5.10 running with the --parallel option dumps the test results out + // to the console with no newlines, so it isn't possible to distinguish where errors + // begin and end. Consequently we can't record them, and so we manually mark them + // as passed or failed here with a manufactured issue. + // Don't attempt to parse the console output of parallel tests between 5.7 and 5.10 + // as it doesn't have newlines. You might get lucky and find the output is split + // in the right spot, but more often than not we wont be able to parse it. + if (!this.hasMultiLineParallelTestOutput) { + return; + } + + // For parallel XCTest runs we get pass/fail results from the xunit XML + // produced at the end of the run, but we still want to monitor the output + // for the individual assertion failures. Wrap the run state and only forward + // along the issues captured during a test run, and let the `TestXUnitParser` + // handle marking tests as completed. + this.outputParser.parseResult(output, new ParallelXCTestRunStateProxy(runState)); + } +} + +/* eslint-disable @typescript-eslint/no-unused-vars */ +class ParallelXCTestRunStateProxy implements ITestRunState { + constructor(private runState: ITestRunState) {} + + getTestItemIndex(id: string, filename: string | undefined): number { + return this.runState.getTestItemIndex(id, filename); + } + recordIssue( + index: number, + message: string | MarkdownString, + location?: Location | undefined + ): void { + this.runState.recordIssue(index, message, location); + } + started(index: number, startTime?: number | undefined): void {} + completed(index: number, timing: { duration: number } | { timestamp: number }): void {} + skipped(index: number): void {} + startedSuite(name: string): void {} + passedSuite(name: string): void {} + failedSuite(name: string): void {} +} +/* eslint-enable @typescript-eslint/no-unused-vars */ + +export class XCTestOutputParser implements IXCTestOutputParser { private regex: TestRegex; /** @@ -93,7 +156,7 @@ export class XCTestOutputParser { lines.pop(); } // if submitted text does not end with a newline then pop that off and store in excess - // for next call of parseResultDarwin + // for next call of parseResult if (output2[output2.length - 1] !== "\n") { runState.excess = lines.pop(); } else { diff --git a/src/TestExplorer/TestRunner.ts b/src/TestExplorer/TestRunner.ts index 39b699bba..bcc62c31b 100644 --- a/src/TestExplorer/TestRunner.ts +++ b/src/TestExplorer/TestRunner.ts @@ -15,43 +15,39 @@ import * as vscode from "vscode"; import * as path from "path"; import * as stream from "stream"; -import * as cp from "child_process"; import * as os from "os"; import * as asyncfs from "fs/promises"; -import { - createXCTestConfiguration, - createSwiftTestConfiguration, - createDarwinTestConfiguration, -} from "../debugger/launch"; import { FolderContext } from "../FolderContext"; -import { - execFile, - execFileStreamOutput, - getErrorDescription, - regexEscapedString, -} from "../utilities/utilities"; -import { getBuildAllTask } from "../tasks/SwiftTaskProvider"; +import { execFile, getErrorDescription } from "../utilities/utilities"; +import { createSwiftTask } from "../tasks/SwiftTaskProvider"; import configuration from "../configuration"; import { WorkspaceContext } from "../WorkspaceContext"; -import { XCTestOutputParser } from "./TestParsers/XCTestOutputParser"; +import { + IXCTestOutputParser, + ParallelXCTestOutputParser, + XCTestOutputParser, +} from "./TestParsers/XCTestOutputParser"; import { SwiftTestingOutputParser } from "./TestParsers/SwiftTestingOutputParser"; -import { Version } from "../utilities/version"; import { LoggingDebugAdapterTracker } from "../debugger/logTracker"; import { TaskOperation } from "../tasks/TaskQueue"; -import { TestXUnitParser, iXUnitTestState } from "./TestXUnitParser"; +import { TestXUnitParser } from "./TestXUnitParser"; import { ITestRunState } from "./TestParsers/TestRunState"; import { TestRunArguments } from "./TestRunArguments"; import { TemporaryFolder } from "../utilities/tempFolder"; import { TestClass, runnableTag, upsertTestItem } from "./TestDiscovery"; +import { TestCoverage } from "../coverage/LcovResults"; +import { TestingDebugConfigurationFactory } from "../debugger/buildConfig"; /** Workspace Folder events */ export enum TestKind { // run tests serially - standard = "standard", + standard = "Standard", // run tests in parallel - parallel = "parallel", + parallel = "Parallel", // run tests and extract test coverage - coverage = "coverage", + coverage = "Coverage", + // run tests and extract test coverage + debug = "Debug", } export enum RunProfileName { @@ -61,16 +57,25 @@ export enum RunProfileName { debug = "Debug Tests", } +export enum TestLibrary { + xctest = "XCTest", + swiftTesting = "swift-testing", +} + export class TestRunProxy { private testRun?: vscode.TestRun; private addedTestItems: { testClass: TestClass; parentIndex: number }[] = []; private runStarted: boolean = false; private queuedOutput: string[] = []; private _testItems: vscode.TestItem[]; + public coverage: TestCoverage; // Allows for introspection on the state of TestItems after a test run. public runState = { - failed: [] as vscode.TestItem[], + failed: [] as { + test: vscode.TestItem; + message: vscode.TestMessage | readonly vscode.TestMessage[]; + }[], passed: [] as vscode.TestItem[], skipped: [] as vscode.TestItem[], errored: [] as vscode.TestItem[], @@ -87,6 +92,7 @@ export class TestRunProxy { private folderContext: FolderContext ) { this._testItems = args.testItems; + this.coverage = new TestCoverage(folderContext); } public testRunStarted = () => { @@ -180,7 +186,7 @@ export class TestRunProxy { message: vscode.TestMessage | readonly vscode.TestMessage[], duration?: number ) { - this.runState.failed.push(test); + this.runState.failed.push({ test, message }); this.testRun?.failed(test, message, duration); } @@ -193,7 +199,7 @@ export class TestRunProxy { this.testRun?.errored(test, message, duration); } - public end() { + public async end() { this.testRun?.end(); } @@ -204,13 +210,22 @@ export class TestRunProxy { this.queuedOutput.push(output); } } + + public async computeCoverage() { + if (!this.testRun) { + return; + } + + // Compute final coverage numbers if any coverage info has been captured during the run. + await this.coverage.computeCoverage(this.testRun); + } } /** Class used to run tests */ export class TestRunner { private testRun: TestRunProxy; private testArgs: TestRunArguments; - private xcTestOutputParser: XCTestOutputParser; + private xcTestOutputParser: IXCTestOutputParser; private swiftTestOutputParser: SwiftTestingOutputParser; /** @@ -220,13 +235,19 @@ export class TestRunner { * @param controller Test controller */ constructor( + private testKind: TestKind, private request: vscode.TestRunRequest, private folderContext: FolderContext, private controller: vscode.TestController ) { this.testArgs = new TestRunArguments(this.ensureRequestIncludesTests(this.request)); this.testRun = new TestRunProxy(request, controller, this.testArgs, folderContext); - this.xcTestOutputParser = new XCTestOutputParser(); + this.xcTestOutputParser = + testKind === TestKind.parallel + ? new ParallelXCTestOutputParser( + this.folderContext.workspaceContext.toolchain.hasMultiLineParallelTestOutput + ) + : new XCTestOutputParser(); this.swiftTestOutputParser = new SwiftTestingOutputParser( this.testRun.testRunStarted, this.testRun.addParameterizedTestCase @@ -266,9 +287,14 @@ export class TestRunner { RunProfileName.run, vscode.TestRunProfileKind.Run, async (request, token) => { - const runner = new TestRunner(request, folderContext, controller); + const runner = new TestRunner( + TestKind.standard, + request, + folderContext, + controller + ); onCreateTestRun.fire(runner.testRun); - await runner.runHandler(false, TestKind.standard, token); + await runner.runHandler(token); }, true, runnableTag @@ -278,9 +304,14 @@ export class TestRunner { RunProfileName.runParallel, vscode.TestRunProfileKind.Run, async (request, token) => { - const runner = new TestRunner(request, folderContext, controller); + const runner = new TestRunner( + TestKind.parallel, + request, + folderContext, + controller + ); onCreateTestRun.fire(runner.testRun); - await runner.runHandler(false, TestKind.parallel, token); + await runner.runHandler(token); }, false, runnableTag @@ -288,11 +319,23 @@ export class TestRunner { // Add coverage profile controller.createRunProfile( RunProfileName.coverage, - vscode.TestRunProfileKind.Run, + vscode.TestRunProfileKind.Coverage, async (request, token) => { - const runner = new TestRunner(request, folderContext, controller); + const runner = new TestRunner( + TestKind.coverage, + request, + folderContext, + controller + ); onCreateTestRun.fire(runner.testRun); - await runner.runHandler(false, TestKind.coverage, token); + if (request.profile) { + request.profile.loadDetailedCoverage = async (testRun, fileCoverage) => { + return runner.testRun.coverage.loadDetailedCoverage(fileCoverage.uri); + }; + } + await runner.runHandler(token); + await runner.testRun.computeCoverage(); + await vscode.commands.executeCommand("testing.openCoverage"); }, false, runnableTag @@ -302,9 +345,14 @@ export class TestRunner { RunProfileName.debug, vscode.TestRunProfileKind.Debug, async (request, token) => { - const runner = new TestRunner(request, folderContext, controller); + const runner = new TestRunner( + TestKind.debug, + request, + folderContext, + controller + ); onCreateTestRun.fire(runner.testRun); - await runner.runHandler(true, TestKind.standard, token); + await runner.runHandler(token); }, false, runnableTag @@ -318,50 +366,24 @@ export class TestRunner { * @param token Cancellation token * @returns When complete */ - async runHandler(shouldDebug: boolean, testKind: TestKind, token: vscode.CancellationToken) { + async runHandler(token: vscode.CancellationToken) { const runState = new TestRunnerTestRunState(this.testRun); try { - // run associated build task - // don't do this if generating code test coverage data as it - // will rebuild everything again - if (testKind !== TestKind.coverage) { - const task = await getBuildAllTask(this.folderContext); - task.definition.dontTriggerTestDiscovery = - this.folderContext.workspaceContext.swiftVersion.isGreaterThanOrEqual( - new Version(6, 0, 0) - ); - - const exitCode = await this.folderContext.taskQueue.queueOperation( - new TaskOperation(task), - token - ); - - // if build failed then exit - if (exitCode === undefined || exitCode !== 0) { - this.testRun.end(); - return; - } - } - - if (shouldDebug) { + if (this.testKind === TestKind.debug) { await this.debugSession(token, runState); } else { - await this.runSession(token, testKind, runState); + await this.runSession(token, runState); } } catch (error) { this.workspaceContext.outputChannel.log(`Error: ${getErrorDescription(error)}`); this.testRun.appendOutput(`\r\nError: ${getErrorDescription(error)}`); } - this.testRun.end(); + await this.testRun.end(); } /** Run test session without attaching to a debugger */ - async runSession( - token: vscode.CancellationToken, - testKind: TestKind, - runState: TestRunnerTestRunState - ) { + async runSession(token: vscode.CancellationToken, runState: TestRunnerTestRunState) { // Run swift-testing first, then XCTest. // swift-testing being parallel by default should help these run faster. if (this.testArgs.hasSwiftTestingTests) { @@ -377,12 +399,13 @@ export class TestRunner { await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext); } - const testBuildConfig = - await LaunchConfigurations.createLaunchConfigurationForSwiftTesting( - this.testArgs.swiftTestArgs, - this.folderContext, - fifoPipePath - ); + const testBuildConfig = TestingDebugConfigurationFactory.swiftTestingConfig( + this.folderContext, + fifoPipePath, + this.testKind, + this.testArgs.swiftTestArgs, + true + ); if (testBuildConfig === null) { return; @@ -407,22 +430,22 @@ export class TestRunner { await this.swiftTestOutputParser.watch(fifoPipePath, runState); await this.launchTests( - testKind === TestKind.parallel ? TestKind.standard : testKind, + runState, + this.testKind === TestKind.parallel ? TestKind.standard : this.testKind, token, outputStream, - outputStream, testBuildConfig, - runState + TestLibrary.swiftTesting ); }); } if (this.testArgs.hasXCTests) { - const testBuildConfig = LaunchConfigurations.createLaunchConfigurationForXCTestTesting( - this.testArgs.xcTestArgs, - this.workspaceContext, + const testBuildConfig = TestingDebugConfigurationFactory.xcTestConfig( this.folderContext, - false + this.testKind, + this.testArgs.xcTestArgs, + true ); if (testBuildConfig === null) { return; @@ -437,18 +460,8 @@ export class TestRunner { }, }); - // Output test from stream - const outputStream = new stream.Writable({ - write: (chunk, encoding, next) => { - const text = chunk.toString(); - this.testRun.appendOutput(text.replace(/\n/g, "\r\n")); - next(); - }, - }); - if (token.isCancellationRequested) { parsedOutputStream.end(); - outputStream.end(); return; } @@ -456,219 +469,172 @@ export class TestRunner { this.testRun.testRunStarted(); await this.launchTests( - testKind, + runState, + this.testKind, token, parsedOutputStream, - outputStream, testBuildConfig, - runState + TestLibrary.xctest ); } } private async launchTests( + runState: TestRunnerTestRunState, testKind: TestKind, token: vscode.CancellationToken, - parsedOutputStream: stream.Writable, outputStream: stream.Writable, testBuildConfig: vscode.DebugConfiguration, - runState: TestRunnerTestRunState + testLibrary: TestLibrary ) { - this.testRun.appendOutput(`> Test run started at ${new Date().toLocaleString()} <\r\n\r\n`); try { switch (testKind) { case TestKind.coverage: await this.runCoverageSession( token, - parsedOutputStream, outputStream, - testBuildConfig + testBuildConfig, + testLibrary ); break; case TestKind.parallel: - await this.runParallelSession( - token, - parsedOutputStream, - outputStream, - testBuildConfig - ); + await this.runParallelSession(token, outputStream, testBuildConfig, runState); break; default: - await this.runStandardSession( - token, - parsedOutputStream, - outputStream, - testBuildConfig - ); + await this.runStandardSession(token, outputStream, testBuildConfig, testKind); break; } } catch (error) { - const execError = error as cp.ExecFileException; - if (execError.code === 1 && execError.killed === false) { - // Process returned an error code probably because a test failed - } else if (execError.killed === true) { - // Process was killed - this.testRun.appendOutput(`\r\nProcess killed.`); - return; - } else if (execError.signal === "SIGILL") { - // Process crashed - this.testRun.appendOutput(`\r\nProcess crashed.`); - if (runState.currentTestItem) { - // get last line of error message, which should include why it crashed - const errorMessagesLines = execError.message.match(/[^\r\n]+/g); - if (errorMessagesLines) { - const message = new vscode.TestMessage( - getErrorDescription(errorMessagesLines[errorMessagesLines.length - 1]) - ); - this.testRun.errored(runState.currentTestItem, message); - } else { - const message = new vscode.TestMessage( - getErrorDescription(execError.message) - ); - this.testRun.errored(runState.currentTestItem, message); - } - } - return; - } else { - // Unrecognised error + // Test failures result in error code 1 + if (error !== 1) { this.testRun.appendOutput(`\r\nError: ${getErrorDescription(error)}`); - return; } } finally { - parsedOutputStream.end(); - if (outputStream !== parsedOutputStream) { - outputStream.end(); - } + outputStream.end(); } } /** Run tests outside of debugger */ async runStandardSession( token: vscode.CancellationToken, - parsedOutputStream: stream.Writable, outputStream: stream.Writable, - testBuildConfig: vscode.DebugConfiguration + testBuildConfig: vscode.DebugConfiguration, + testKind: TestKind ) { - // Darwin outputs XCTest output to stderr, Linux outputs XCTest output to stdout - let stdout: stream.Writable; - let stderr: stream.Writable; - if (process.platform === "darwin") { - stdout = outputStream; - stderr = parsedOutputStream; - } else { - stdout = parsedOutputStream; - stderr = outputStream; - } + return new Promise((resolve, reject) => { + const args = testBuildConfig.args ?? []; + this.folderContext?.workspaceContext.outputChannel.logDiagnostic( + `Exec: ${testBuildConfig.program} ${args.join(" ")}`, + this.folderContext.name + ); - await execFileStreamOutput( - testBuildConfig.program, - testBuildConfig.args ?? [], - stdout, - stderr, - token, - { - cwd: testBuildConfig.cwd, - env: { ...process.env, ...testBuildConfig.env }, - maxBuffer: 16 * 1024 * 1024, - }, - this.folderContext, - false, - "SIGKILL" - ); + let kindLabel: string; + switch (testKind) { + case TestKind.coverage: + kindLabel = " With Code Coverage"; + break; + case TestKind.parallel: + kindLabel = " In Parallel"; + break; + case TestKind.debug: + kindLabel = "For Debugging"; + break; + case TestKind.standard: + kindLabel = ""; + } + + const task = createSwiftTask( + args, + `Building and Running Tests${kindLabel}`, + { + cwd: this.folderContext.folder, + scope: this.folderContext.workspaceFolder, + prefix: this.folderContext.name, + presentationOptions: { reveal: vscode.TaskRevealKind.Silent }, + }, + this.folderContext.workspaceContext.toolchain, + { ...process.env, ...testBuildConfig.env } + ); + + task.execution.onDidWrite(str => { + const replaced = str + .replace("[1/1] Planning build", "") // Work around SPM still emitting progress when doing --no-build. + .replace( + /LLVM Profile Error: Failed to write file "default.profraw": Operation not permitted\r\n/gm, + "" + ); // Work around benign LLVM coverage warnings + outputStream.write(replaced); + }); + + let cancellation: vscode.Disposable; + task.execution.onDidClose(code => { + if (cancellation) { + cancellation.dispose(); + } + + // undefined or 0 are viewed as success + if (!code) { + resolve(); + } else { + reject(code); + } + }); + + this.folderContext.taskQueue.queueOperation(new TaskOperation(task), token); + }); } /** Run tests with code coverage, and parse coverage results */ async runCoverageSession( token: vscode.CancellationToken, - stdout: stream.Writable, - stderr: stream.Writable, - testBuildConfig: vscode.DebugConfiguration + outputStream: stream.Writable, + testBuildConfig: vscode.DebugConfiguration, + testLibrary: TestLibrary ) { try { - // XCTestRuns are started immediately - this.testRun.testRunStarted(); - - // TODO: This approach only covers xctests. - const filterArgs = this.testArgs.xcTestArgs.flatMap(arg => ["--filter", arg]); - const args = ["test", "--enable-code-coverage"]; - await execFileStreamOutput( - this.workspaceContext.toolchain.getToolchainExecutable("swift"), - [...args, ...filterArgs], - stdout, - stderr, - token, - { - cwd: testBuildConfig.cwd, - env: { ...process.env, ...testBuildConfig.env, SWT_SF_SYMBOLS_ENABLED: "0" }, - maxBuffer: 16 * 1024 * 1024, - }, - this.folderContext, - false, - "SIGINT" // use SIGINT to kill process as it is a child process of `swift test` - ); + await this.runStandardSession(token, outputStream, testBuildConfig, TestKind.coverage); } catch (error) { - const execError = error as cp.ExecFileException; - if (execError.code !== 1 || execError.killed === true) { + // If this isn't a standard test failure, forward the error and skip generating coverage. + if (error !== 1) { throw error; } } - await this.folderContext.lcovResults.generate(); - if (configuration.displayCoverageReportAfterRun) { - this.workspaceContext.testCoverageDocumentProvider.show(this.folderContext); - } + + await this.testRun.coverage.captureCoverage(testLibrary); } /** Run tests in parallel outside of debugger */ async runParallelSession( token: vscode.CancellationToken, - stdout: stream.Writable, - stderr: stream.Writable, - testBuildConfig: vscode.DebugConfiguration + outputStream: stream.Writable, + testBuildConfig: vscode.DebugConfiguration, + runState: TestRunnerTestRunState ) { await this.workspaceContext.tempFolder.withTemporaryFile("xml", async filename => { - const sanitizer = this.workspaceContext.toolchain.sanitizer(configuration.sanitizer); - const sanitizerArgs = sanitizer?.buildFlags ?? []; - const filterArgs = this.testArgs.xcTestArgs.flatMap(arg => ["--filter", arg]); - const args = [ - "test", - "--parallel", - ...sanitizerArgs, - "--skip-build", - "--xunit-output", - filename, - ]; - - // XCTestRuns are started immediately - this.testRun.testRunStarted(); + const args = [...(testBuildConfig.args ?? []), "--xunit-output", filename]; try { - await execFileStreamOutput( - this.workspaceContext.toolchain.getToolchainExecutable("swift"), - [...args, ...filterArgs], - stdout, - stderr, + testBuildConfig.args = await this.runStandardSession( token, + outputStream, { - cwd: testBuildConfig.cwd, - env: { ...process.env, ...testBuildConfig.env }, - maxBuffer: 16 * 1024 * 1024, + ...testBuildConfig, + args, }, - this.folderContext, - false, - "SIGINT" // use SIGINT to kill process as it is a child process of `swift test` + TestKind.parallel ); } catch (error) { - const execError = error as cp.ExecFileException; - if (execError.code !== 1 || execError.killed === true) { + // If this isn't a standard test failure, forward the error and skip generating coverage. + if (error !== 1) { throw error; } } + const buffer = await asyncfs.readFile(filename, "utf8"); - const xUnitParser = new TestXUnitParser(); - const results = await xUnitParser.parse( - buffer, - new TestRunnerXUnitTestState(this.testItemFinder, this.testRun) + const xUnitParser = new TestXUnitParser( + this.folderContext.workspaceContext.toolchain.hasMultiLineParallelTestOutput ); + const results = await xUnitParser.parse(buffer, runState); if (results) { this.testRun.appendOutput( `\r\nExecuted ${results.tests} tests, with ${results.failures} failures and ${results.errors} errors.\r\n` @@ -694,17 +660,23 @@ export class TestRunner { } if (this.testArgs.hasSwiftTestingTests) { - const swiftTestBuildConfig = - await LaunchConfigurations.createLaunchConfigurationForSwiftTesting( - this.testArgs.swiftTestArgs, - this.folderContext, - fifoPipePath - ); + const swiftTestBuildConfig = TestingDebugConfigurationFactory.swiftTestingConfig( + this.folderContext, + fifoPipePath, + TestKind.debug, + this.testArgs.swiftTestArgs, + true + ); if (swiftTestBuildConfig !== null) { - // given we have already run a build task there is no need to have a pre launch task - // to build the tests - swiftTestBuildConfig.preLaunchTask = undefined; + // output test build configuration + if (configuration.diagnostics) { + const configJSON = JSON.stringify(swiftTestBuildConfig); + this.workspaceContext.outputChannel.logDiagnostic( + `swift-testing Debug Config: ${configJSON}`, + this.folderContext.name + ); + } // output test build configuration if (configuration.diagnostics) { @@ -718,25 +690,21 @@ export class TestRunner { // The await simply waits for the watching to be configured. await this.swiftTestOutputParser.watch(fifoPipePath, runState); + swiftTestBuildConfig.testType = TestLibrary.swiftTesting; buildConfigs.push(swiftTestBuildConfig); } } // create launch config for testing if (this.testArgs.hasXCTests) { - const xcTestBuildConfig = - await LaunchConfigurations.createLaunchConfigurationForXCTestTesting( - this.testArgs.xcTestArgs, - this.workspaceContext, - this.folderContext, - true - ); + const xcTestBuildConfig = TestingDebugConfigurationFactory.xcTestConfig( + this.folderContext, + TestKind.debug, + this.testArgs.xcTestArgs, + true + ); if (xcTestBuildConfig !== null) { - // given we have already run a build task there is no need to have a pre launch task - // to build the tests - xcTestBuildConfig.preLaunchTask = undefined; - // output test build configuration if (configuration.diagnostics) { const configJSON = JSON.stringify(xcTestBuildConfig); @@ -746,6 +714,8 @@ export class TestRunner { ); } + xcTestBuildConfig.testType = TestLibrary.xctest; + buildConfigs.push(xcTestBuildConfig); } } @@ -761,13 +731,19 @@ export class TestRunner { new Promise((resolve, reject) => { // add cancelation const startSession = vscode.debug.onDidStartDebugSession(session => { + if (config.testType === TestLibrary.xctest) { + this.testRun.testRunStarted(); + } + this.workspaceContext.outputChannel.logDiagnostic( "Start Test Debugging", this.folderContext.name ); LoggingDebugAdapterTracker.setDebugSessionCallback(session, output => { this.testRun.appendOutput(output); - this.xcTestOutputParser.parseResult(output, runState); + if (config.testType === TestLibrary.xctest) { + this.xcTestOutputParser.parseResult(output, runState); + } }); const cancellation = token.onCancellationRequested(() => { this.workspaceContext.outputChannel.logDiagnostic( @@ -785,11 +761,6 @@ export class TestRunner { .then( started => { if (started) { - if (config === validBuildConfigs[0]) { - this.testRun.appendOutput( - `> Test run started at ${new Date().toLocaleString()} <\r\n\r\n` - ); - } // show test results pane vscode.commands.executeCommand( "testing.showMostRecentOutput" @@ -839,112 +810,6 @@ export class TestRunner { } } -class LaunchConfigurations { - /** - * Edit launch configuration to run tests - * @param debugging Do we need this configuration for debugging - * @param outputFile Debug output file - * @returns - */ - static createLaunchConfigurationForXCTestTesting( - args: string[], - workspaceContext: WorkspaceContext, - folderContext: FolderContext, - debugging: boolean - ): vscode.DebugConfiguration | null { - const testList = args.join(","); - - if (process.platform === "darwin") { - // if debugging on macOS with Swift 5.6 we need to create a custom launch - // configuration so we can set the system architecture - const swiftVersion = workspaceContext.toolchain.swiftVersion; - if ( - debugging && - swiftVersion.isLessThan(new Version(5, 7, 0)) && - swiftVersion.isGreaterThanOrEqual(new Version(5, 6, 0)) - ) { - let testFilterArg: string; - if (testList.length > 0) { - testFilterArg = `-XCTest ${testList}`; - } else { - testFilterArg = ""; - } - const testBuildConfig = createDarwinTestConfiguration(folderContext, testFilterArg); - if (testBuildConfig === null) { - return null; - } - return testBuildConfig; - } else { - const testBuildConfig = createXCTestConfiguration(folderContext, true); - if (testBuildConfig === null) { - return null; - } - - if (testList.length > 0) { - testBuildConfig.args = ["-XCTest", testList, ...testBuildConfig.args]; - } - - // output test logging to debug console so we can catch it with a tracker - testBuildConfig.terminal = "console"; - return testBuildConfig; - } - } else { - const testBuildConfig = createXCTestConfiguration(folderContext, true); - if (testBuildConfig === null) { - return null; - } - - if (testList.length > 0) { - testBuildConfig.args = [testList]; - } - // output test logging to debug console so we can catch it with a tracker - testBuildConfig.terminal = "console"; - return testBuildConfig; - } - } - - static async createLaunchConfigurationForSwiftTesting( - args: string[], - folderContext: FolderContext, - fifoPipePath: string - ): Promise { - const testList = args.join(","); - - if (process.platform === "darwin") { - const testBuildConfig = createSwiftTestConfiguration(folderContext, fifoPipePath, true); - if (testBuildConfig === null) { - return null; - } - - let testFilterArg: string[] = []; - if (testList.length > 0) { - testFilterArg = args.flatMap(arg => ["--filter", regexEscapedString(arg)]); - } - - testBuildConfig.args = [...testBuildConfig.args, ...testFilterArg]; - testBuildConfig.terminal = "console"; - - return testBuildConfig; - } else { - const testBuildConfig = createSwiftTestConfiguration(folderContext, fifoPipePath, true); - if (testBuildConfig === null) { - return null; - } - - let testFilterArg: string[] = []; - if (testList.length > 0) { - testFilterArg = args.flatMap(arg => ["--filter", regexEscapedString(arg)]); - } - - testBuildConfig.args = [...testBuildConfig.args, ...testFilterArg]; - - // output test logging to debug console so we can catch it with a tracker - testBuildConfig.terminal = "console"; - return testBuildConfig; - } - } -} - /** Interface defining how to find test items given a test id from XCTest output */ interface TestItemFinder { getIndex(id: string, filename?: string): number; @@ -1025,7 +890,7 @@ class NonDarwinTestItemFinder implements TestItemFinder { /** * Store state of current test run output parse */ -class TestRunnerTestRunState implements ITestRunState { +export class TestRunnerTestRunState implements ITestRunState { constructor(private testRun: TestRunProxy) {} public currentTestItem?: vscode.TestItem; @@ -1115,30 +980,3 @@ class TestRunnerTestRunState implements ITestRunState { // Nothing to do here } } - -class TestRunnerXUnitTestState implements iXUnitTestState { - constructor( - private testItemFinder: TestItemFinder, - private testRun: TestRunProxy - ) {} - - passTest(id: string, duration: number): void { - const index = this.testItemFinder.getIndex(id); - if (index !== -1) { - this.testRun.passed(this.testItemFinder.testItems[index], duration); - } - } - failTest(id: string, duration: number, message?: string): void { - const index = this.testItemFinder.getIndex(id); - if (index !== -1) { - const testMessage = new vscode.TestMessage(message ?? "Failed"); - this.testRun.failed(this.testItemFinder.testItems[index], testMessage, duration); - } - } - skipTest(id: string): void { - const index = this.testItemFinder.getIndex(id); - if (index !== -1) { - this.testRun.skipped(this.testItemFinder.testItems[index]); - } - } -} diff --git a/src/TestExplorer/TestXUnitParser.ts b/src/TestExplorer/TestXUnitParser.ts index 952ee55c3..4737fdf1d 100644 --- a/src/TestExplorer/TestXUnitParser.ts +++ b/src/TestExplorer/TestXUnitParser.ts @@ -13,12 +13,7 @@ //===----------------------------------------------------------------------===// import * as xml2js from "xml2js"; - -export interface iXUnitTestState { - passTest(id: string, duration: number): void; - failTest(id: string, duration: number, message?: string): void; - skipTest(id: string): void; -} +import { TestRunnerTestRunState } from "./TestRunner"; export interface TestResults { tests: number; @@ -27,7 +22,7 @@ export interface TestResults { } interface XUnitFailure { - message?: string; + $: { message?: string }; } interface XUnitTestCase { @@ -49,9 +44,12 @@ interface XUnit { } export class TestXUnitParser { - constructor() {} + constructor(private hasMultiLineParallelTestOutput: boolean) {} - async parse(buffer: string, runState: iXUnitTestState): Promise { + async parse( + buffer: string, + runState: TestRunnerTestRunState + ): Promise { const xml = await xml2js.parseStringPromise(buffer); try { return await this.parseXUnit(xml, runState); @@ -63,7 +61,7 @@ export class TestXUnitParser { } // eslint-disable-next-line @typescript-eslint/no-unused-vars - async parseXUnit(xUnit: XUnit, runState: iXUnitTestState): Promise { + async parseXUnit(xUnit: XUnit, runState: TestRunnerTestRunState): Promise { let tests = 0; let failures = 0; let errors = 0; @@ -73,10 +71,21 @@ export class TestXUnitParser { errors += parseInt(testsuite.$.errors); testsuite.testcase.forEach(testcase => { const id = `${testcase.$.classname}/${testcase.$.name}`; - if (testcase.failure) { - runState.failTest(id, testcase.$.time, testcase.failure.shift()?.message); - } else { - runState.passTest(id, testcase.$.time); + const index = runState.getTestItemIndex(id); + + if (index !== -1) { + // From 5.7 to 5.10 running with the --parallel option dumps the test results out + // to the console with no newlines, so it isn't possible to distinguish where errors + // begin and end. Consequently we can't record them, and so we manually mark them + // as passed or failed here with a manufactured issue. + if (!!testcase.failure && !this.hasMultiLineParallelTestOutput) { + runState.recordIssue( + index, + testcase.failure.shift()?.$.message ?? "Test Failed" + ); + } + + runState.completed(index, { duration: testcase.$.time }); } }); }); diff --git a/src/WorkspaceContext.ts b/src/WorkspaceContext.ts index cecf7e605..36854b454 100644 --- a/src/WorkspaceContext.ts +++ b/src/WorkspaceContext.ts @@ -33,9 +33,7 @@ import { makeDebugConfigurations } from "./debugger/launch"; import configuration from "./configuration"; import contextKeys from "./contextKeys"; import { setSnippetContextKey } from "./SwiftSnippets"; -import { TestCoverageReportProvider } from "./coverage/TestCoverageReport"; import { CommentCompletionProviders } from "./editor/CommentCompletion"; -import { TestCoverageRenderer } from "./coverage/TestCoverageRenderer"; import { DebugAdapter } from "./debugger/debugAdapter"; import { Version } from "./utilities/version"; import { SwiftBuildStatus } from "./ui/SwiftBuildStatus"; @@ -54,9 +52,7 @@ export class WorkspaceContext implements vscode.Disposable { public languageClientManager: LanguageClientManager; public tasks: TaskManager; public subscriptions: { dispose(): unknown }[]; - public testCoverageDocumentProvider: TestCoverageReportProvider; public commentCompletionProvider: CommentCompletionProviders; - public testCoverageRenderer: TestCoverageRenderer; private lastFocusUri: vscode.Uri | undefined; private initialisationFinished = false; @@ -72,10 +68,7 @@ export class WorkspaceContext implements vscode.Disposable { this.toolchain.logDiagnostics(this.outputChannel); this.tasks = new TaskManager(this); this.currentDocument = null; - // test coverage document provider - this.testCoverageDocumentProvider = new TestCoverageReportProvider(this); this.commentCompletionProvider = new CommentCompletionProviders(); - this.testCoverageRenderer = new TestCoverageRenderer(this); contextKeys.createNewProjectAvailable = toolchain.swiftVersion.isGreaterThanOrEqual( new Version(5, 8, 0) ); @@ -213,8 +206,6 @@ export class WorkspaceContext implements vscode.Disposable { swiftFileWatcher, onDidEndTask, this.commentCompletionProvider, - this.testCoverageDocumentProvider, - this.testCoverageRenderer, backgroundCompilationOnDidSave, contextKeysUpdate, onChangeConfig, @@ -587,14 +578,6 @@ export class WorkspaceContext implements vscode.Disposable { } } - public toggleTestCoverageDisplay() { - if (!this.testCoverageRenderer) { - this.testCoverageRenderer = new TestCoverageRenderer(this); - this.subscriptions.push(this.testCoverageRenderer); - } - this.testCoverageRenderer.toggleDisplayResults(); - } - private initialisationComplete() { this.initialisationFinished = true; if (this.lastFocusUri) { diff --git a/src/commands.ts b/src/commands.ts index 2dd1c25a3..74d350e72 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -778,17 +778,6 @@ async function selectXcodeDeveloperDir() { ); } -export async function showTestCoverageReport(workspaceContext: WorkspaceContext) { - // show test coverage report - if (workspaceContext.currentFolder) { - workspaceContext.testCoverageDocumentProvider.show(workspaceContext.currentFolder); - } -} - -function toggleTestCoverageDisplay(workspaceContext: WorkspaceContext) { - workspaceContext.toggleTestCoverageDisplay(); -} - async function attachDebugger(workspaceContext: WorkspaceContext) { // use LLDB to get list of processes const lldb = workspaceContext.toolchain.getLLDB(); @@ -859,12 +848,6 @@ export function register(ctx: WorkspaceContext) { vscode.commands.registerCommand("swift.insertFunctionComment", () => insertFunctionComment(ctx) ), - vscode.commands.registerCommand("swift.showTestCoverageReport", () => - showTestCoverageReport(ctx) - ), - vscode.commands.registerCommand("swift.toggleTestCoverage", () => - toggleTestCoverageDisplay(ctx) - ), vscode.commands.registerCommand("swift.useLocalDependency", item => { if (item instanceof PackageNode) { useLocalDependency(item.name, ctx); diff --git a/src/coverage/LcovResults.ts b/src/coverage/LcovResults.ts index 8c0541c5e..21ceefc80 100644 --- a/src/coverage/LcovResults.ts +++ b/src/coverage/LcovResults.ts @@ -2,7 +2,7 @@ // // This source file is part of the VSCode Swift open source project // -// Copyright (c) 2021-2022 the VSCode Swift project authors +// Copyright (c) 2024 the VSCode Swift project authors // Licensed under Apache License v2.0 // // See LICENSE.txt for license information @@ -14,136 +14,182 @@ import * as vscode from "vscode"; import * as lcov from "lcov-parse"; -import * as fs from "fs"; import * as asyncfs from "fs/promises"; +import { Writable } from "stream"; +import { promisify } from "util"; import configuration from "../configuration"; import { FolderContext } from "../FolderContext"; import { execFileStreamOutput } from "../utilities/utilities"; import { BuildFlags } from "../toolchain/BuildFlags"; +import { TestLibrary } from "../TestExplorer/TestRunner"; +import { DisposableFileCollection } from "../utilities/tempFolder"; -/** - * Class keeping a record of the latest test coverage results for a package - */ -export class LcovResults implements vscode.Disposable { - public contents: lcov.LcovFile[] | undefined; - public observer: ((results: LcovResults) => unknown) | undefined; +interface CodeCovFile { + testLibrary: TestLibrary; + path: string; +} + +export class TestCoverage { + private lcovFiles: CodeCovFile[] = []; + private lcovTmpFiles: DisposableFileCollection; + private coverageDetails = new Map(); - constructor(public folderContext: FolderContext) { - this.load(); + constructor(private folderContext: FolderContext) { + const tmpFolder = folderContext.workspaceContext.tempFolder; + this.lcovTmpFiles = tmpFolder.createDisposableFileCollection(); } - dispose() { - this.observer = undefined; + /** + * Returns coverage information for the suppplied URI. + */ + public loadDetailedCoverage(uri: vscode.Uri) { + return this.coverageDetails.get(uri) || []; } /** - * Generate LCOV file from profdata output by `swift test --enable-code-coverage`. Then - * load these results into the contents. + * Captures the coverage data after an individual test binary has been run. + * After the test run completes then the coverage is merged. */ - async generate() { - const llvmCov = - this.folderContext.workspaceContext.toolchain.getToolchainExecutable("llvm-cov"); - const packageName = this.folderContext.swiftPackage.name; + public async captureCoverage(testLibrary: TestLibrary) { const buildDirectory = BuildFlags.buildDirectoryFromWorkspacePath( this.folderContext.folder.fsPath, true ); - const lcovFileName = `${buildDirectory}/debug/codecov/lcov.info`; + const result = await asyncfs.readFile(`${buildDirectory}/debug/codecov/default.profdata`); + const filename = this.lcovTmpFiles.file(testLibrary, "profdata"); + await asyncfs.writeFile(filename, result); + this.lcovFiles.push({ testLibrary, path: filename }); + } - // Use WriteStream to log results - const lcovStream = fs.createWriteStream(lcovFileName); + /** + * Once all test binaries have been run compute the coverage information and + * associate it with the test run. + */ + async computeCoverage(testRun: vscode.TestRun) { + const lcovFiles = await this.computeLCOVCoverage(); + if (lcovFiles.length > 0) { + for (const sourceFileCoverage of lcovFiles) { + const uri = vscode.Uri.file(sourceFileCoverage.file); + const detailedCoverage: vscode.FileCoverageDetail[] = []; + for (const lineCoverage of sourceFileCoverage.lines.details) { + const statementCoverage = new vscode.StatementCoverage( + lineCoverage.hit, + new vscode.Position(lineCoverage.line - 1, 0) + ); + detailedCoverage.push(statementCoverage); + } - try { - let xctestFile = `${buildDirectory}/debug/${packageName}PackageTests.xctest`; - if (process.platform === "darwin") { - xctestFile += `/Contents/MacOs/${packageName}PackageTests`; + const coverage = vscode.FileCoverage.fromDetails(uri, detailedCoverage); + testRun.addCoverage(coverage); + this.coverageDetails.set(uri, detailedCoverage); } - await execFileStreamOutput( - llvmCov, - [ - "export", - "--format", - "lcov", - xctestFile, - "--ignore-filename-regex=Tests|.build|Snippets|Plugins", - `--instr-profile=${buildDirectory}/debug/codecov/default.profdata`, - ], - lcovStream, - lcovStream, - null, - { - env: { ...process.env, ...configuration.swiftEnvironmentVariables }, - maxBuffer: 16 * 1024 * 1024, - }, - this.folderContext - ); - await this.lcovFileChanged(); - } catch (error) { - lcovStream.end(); - throw error; } - } - - get exist(): boolean { - return this.contents !== undefined; + this.lcovTmpFiles.dispose(); } /** - * Get the code coverage results for a specified file - * @param filename File we want code coverage data for - * @returns Code coverage results + * Merges multiple `.profdata` files into a single `.profdata` file. */ - resultsForFile(filename: string): lcov.LcovFile | undefined { - return this.contents?.find(item => item.file === filename); - } + private async mergeProfdata(profDataFiles: string[]) { + const filename = this.lcovTmpFiles.file("merged", "profdata"); + const toolchain = this.folderContext.workspaceContext.toolchain; + const llvmProfdata = toolchain.getToolchainExecutable("llvm-profdata"); + await execFileStreamOutput( + llvmProfdata, + ["merge", "-sparse", "-o", filename, ...profDataFiles], + null, + null, + null, + { + env: process.env, + maxBuffer: 16 * 1024 * 1024, + }, + this.folderContext + ); - get totals(): { hit: number; found: number } | undefined { - if (!this.contents) { - return undefined; - } - let hit = 0; - let found = 0; - this.contents.forEach(file => { - hit += file.lines.hit; - found += file.lines.found; - }); - return { hit: hit, found: found }; + return filename; } - private async lcovFileChanged() { - await this.load(); - if (this.observer) { - this.observer(this); + private async computeLCOVCoverage(): Promise { + if (this.lcovFiles.length === 0) { + return []; } - } - private async load() { - const lcovFile = this.lcovFilename(); try { - const buffer = await asyncfs.readFile(lcovFile, "utf8"); - this.contents = await this.loadLcov(buffer); - } catch { - // LCOV file failed to load, but that's ok - } - } + // Merge all the profdata files from each test binary. + const mergedProfileFile = await this.mergeProfdata( + this.lcovFiles.map(({ path }) => path) + ); - private async loadLcov(lcovContents: string): Promise { - return new Promise((resolve, reject) => { - lcov.source(lcovContents, (error, data) => { - if (error) { - reject(error); - } else if (data) { - resolve(data); - } - }); - }); + // Then export to the final lcov file that + // can be processed and fed to VSCode. + const lcovData = await this.exportProfdata( + this.lcovFiles.map(({ testLibrary }) => testLibrary), + mergedProfileFile + ); + + return await this.loadLcov(lcovData.toString("utf8")); + } catch (error) { + return []; + } } - private lcovFilename() { + /** + * Exports a `.profdata` file using `llvm-cov export`, returning the result as a `Buffer`. + */ + private async exportProfdata(types: TestLibrary[], mergedProfileFile: string): Promise { + const packageName = this.folderContext.swiftPackage.name; const buildDirectory = BuildFlags.buildDirectoryFromWorkspacePath( this.folderContext.folder.fsPath, true ); - return `${buildDirectory}/debug/codecov/lcov.info`; + + const coveredBinaries: string[] = []; + if (types.includes(TestLibrary.xctest)) { + let xcTestBinary = `${buildDirectory}/debug/${packageName}PackageTests.xctest`; + if (process.platform === "darwin") { + xcTestBinary += `/Contents/MacOS/${packageName}PackageTests`; + } + coveredBinaries.push(xcTestBinary); + } + + if (types.includes(TestLibrary.swiftTesting)) { + const swiftTestBinary = `${buildDirectory}/debug/${packageName}PackageTests.swift-testing`; + coveredBinaries.push(swiftTestBinary); + } + + let buffer = Buffer.alloc(0); + const writableStream = new Writable({ + write(chunk, encoding, callback) { + buffer = Buffer.concat([buffer, chunk]); + callback(); + }, + }); + + await execFileStreamOutput( + this.folderContext.workspaceContext.toolchain.getToolchainExecutable("llvm-cov"), + [ + "export", + "--format", + "lcov", + ...coveredBinaries, + "--ignore-filename-regex=Tests|swift-testing|Testing|.build|Snippets|Plugins", + `--instr-profile=${mergedProfileFile}`, + ], + writableStream, + writableStream, + null, + { + env: { ...process.env, ...configuration.swiftEnvironmentVariables }, + maxBuffer: 16 * 1024 * 1024, + }, + this.folderContext + ); + + return buffer; + } + + private async loadLcov(lcovContents: string): Promise { + return promisify(lcov.source)(lcovContents).then(value => value ?? []); } } diff --git a/src/coverage/TestCoverageRenderer.ts b/src/coverage/TestCoverageRenderer.ts deleted file mode 100644 index 17d29b41e..000000000 --- a/src/coverage/TestCoverageRenderer.ts +++ /dev/null @@ -1,281 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the VSCode Swift open source project -// -// Copyright (c) 2021-2022 the VSCode Swift project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of VSCode Swift project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import * as vscode from "vscode"; -import configuration from "../configuration"; -import { FolderEvent, WorkspaceContext } from "../WorkspaceContext"; -import { LcovResults } from "./LcovResults"; - -export class TestCoverageRenderer implements vscode.Disposable { - private displayResults: boolean; - private subscriptions: { dispose(): unknown }[]; - private currentEditor: vscode.TextEditor | undefined; - private coverageHitDecorationType: vscode.TextEditorDecorationType; - private coverageMissDecorationType: vscode.TextEditorDecorationType; - private statusBarItem: vscode.StatusBarItem; - - constructor(private workspaceContext: WorkspaceContext) { - this.displayResults = false; - this.currentEditor = vscode.window.activeTextEditor; - - // decoration types for hit and missed lines of code - const { hit, miss } = this.getTestCoverageDecorationTypes(); - this.coverageHitDecorationType = vscode.window.createTextEditorDecorationType(hit); - this.coverageMissDecorationType = vscode.window.createTextEditorDecorationType(miss); - - // status bar item displaying percentage of coverage for the current file - this.statusBarItem = this.createCoverageStatusItem(); - - // set observer on all currently loaded folders lcov results - workspaceContext.folders.forEach(folder => { - folder.lcovResults.observer = results => { - this.resultsChanged(results); - }; - }); - // whenever a new folder is added set observer on lcov results - const folderAddedObserver = workspaceContext.observeFolders((folder, event) => { - if (!folder) { - return; - } - switch (event) { - case FolderEvent.add: - folder.lcovResults.observer = results => { - this.resultsChanged(results); - }; - } - }); - // add event listener for when the active edited text document changes - const onDidChangeActiveWindow = vscode.window.onDidChangeActiveTextEditor(async editor => { - if (this.currentEditor) { - this.clear(this.currentEditor); - } - if (editor) { - this.render(editor); - this.currentEditor = editor; - } - }); - // on configuration change rebuild test coverage decorations with new colors - const onChangeConfig = vscode.workspace.onDidChangeConfiguration(event => { - if (event.affectsConfiguration("swift.coverage.colors")) { - this.resetTestCoverageEditorColors(); - } - if (event.affectsConfiguration("swift.coverage.alwaysShowStatusItem")) { - this.updateCoverageStatusItem(); - } - }); - this.subscriptions = [folderAddedObserver, onDidChangeActiveWindow, onChangeConfig]; - } - - dispose() { - this.subscriptions.forEach(item => item.dispose()); - this.coverageHitDecorationType.dispose(); - this.coverageMissDecorationType.dispose(); - } - - private createCoverageStatusItem(): vscode.StatusBarItem { - // status bar item displaying percentage of coverage for the current file - const statusBarItem = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left); - if (configuration.alwaysShowCoverageStatusItem) { - statusBarItem.text = "Coverage: Off"; - statusBarItem.accessibilityInformation = { - label: "Coverage: Off", - role: "button", - }; - statusBarItem.command = "swift.toggleTestCoverage"; - statusBarItem.show(); - } - return statusBarItem; - } - - /** Update coverage status bar item after configuration has changed */ - private updateCoverageStatusItem() { - if (configuration.alwaysShowCoverageStatusItem) { - this.updateCoverageStatusItemText(this.statusItemCoverageOffText()); - this.statusBarItem.command = "swift.toggleTestCoverage"; - this.statusBarItem.show(); - } else { - this.statusBarItem.command = undefined; - this.statusBarItem.hide(); - } - } - - /** Update text and accessibility data for test coverage status item */ - private updateCoverageStatusItemText(text: string) { - this.statusBarItem.text = text; - this.statusBarItem.accessibilityInformation = { - label: text, - role: configuration.alwaysShowCoverageStatusItem ? "button" : undefined, - }; - } - - /** Reset test coverage colors. Most likely because they have been edited in the settings */ - private resetTestCoverageEditorColors() { - if (this.currentEditor) { - this.clear(this.currentEditor); - } - this.coverageHitDecorationType.dispose(); - this.coverageMissDecorationType.dispose(); - const { hit, miss } = this.getTestCoverageDecorationTypes(); - this.coverageHitDecorationType = vscode.window.createTextEditorDecorationType(hit); - this.coverageMissDecorationType = vscode.window.createTextEditorDecorationType(miss); - } - - /** Return decoration render options for hit and miss decorations */ - private getTestCoverageDecorationTypes(): { - hit: vscode.DecorationRenderOptions; - miss: vscode.DecorationRenderOptions; - } { - const hitDecorationType: vscode.DecorationRenderOptions = { - isWholeLine: true, - dark: { - backgroundColor: configuration.coverageHitColorDarkMode, - overviewRulerColor: configuration.coverageHitColorDarkMode, - }, - light: { - backgroundColor: configuration.coverageHitColorLightMode, - overviewRulerColor: configuration.coverageHitColorLightMode, - }, - }; - const missDecorationType: vscode.DecorationRenderOptions = { - isWholeLine: true, - dark: { - backgroundColor: configuration.coverageMissColorDarkMode, - overviewRulerColor: configuration.coverageMissColorDarkMode, - }, - light: { - backgroundColor: configuration.coverageMissColorLightMode, - overviewRulerColor: configuration.coverageMissColorLightMode, - }, - }; - return { hit: hitDecorationType, miss: missDecorationType }; - } - - /** - * Toggle display of coverage results - */ - toggleDisplayResults() { - if (this.displayResults === true) { - this.displayResults = false; - if (this.currentEditor) { - this.clear(this.currentEditor); - } else { - this.updateCoverageStatusItemText(this.statusItemCoverageOffText()); - } - } else { - this.displayResults = true; - if (this.currentEditor) { - this.render(this.currentEditor); - } else { - this.updateCoverageStatusItemText(this.statusItemCoverageOffText()); - } - } - } - - private resultsChanged(results: LcovResults) { - if (results.folderContext === this.workspaceContext.currentFolder && this.currentEditor) { - this.render(this.currentEditor); - } - } - - private render(editor: vscode.TextEditor) { - // clear previous results - this.clear(editor); - - const folder = this.workspaceContext.currentFolder; - if (!folder || !this.displayResults) { - return; - } - - if (!folder.lcovResults.exist) { - vscode.window.showInformationMessage("Test coverage results are unavailable."); - this.displayResults = false; - } - - const results = folder.lcovResults.resultsForFile(editor?.document.fileName); - if (!results) { - return; - } - const hits = results.lines.details.filter(line => line.hit > 0); - const misses = results.lines.details.filter(line => line.hit === 0); - if (hits.length > 0) { - const ranges = hits.map(line => { - return new vscode.Range( - new vscode.Position(line.line - 1, 0), - new vscode.Position(line.line - 1, 0) - ); - }); - const combinedRanges = this.combineRanges(ranges); - editor.setDecorations(this.coverageHitDecorationType, combinedRanges); - } - if (misses.length > 0) { - const ranges = misses.map(line => { - return new vscode.Range( - new vscode.Position(line.line - 1, 0), - new vscode.Position(line.line - 1, 0) - ); - }); - const combinedRanges = this.combineRanges(ranges); - editor.setDecorations(this.coverageMissDecorationType, combinedRanges); - } - - const coveragePercentage = (100.0 * results.lines.hit) / results.lines.found; - this.updateCoverageStatusItemText(`Coverage: ${coveragePercentage.toFixed(1)}%`); - this.statusBarItem.show(); - } - - /** - * Combine any ranges that are next to each other - * @param ranges List of ranges - * @returns Combined ranges - */ - combineRanges(ranges: vscode.Range[]): vscode.Range[] { - let lastRange = ranges[0]; - const combinedRanges: vscode.Range[] = []; - // if ranges length is less than 2 there aren't any ranges to combine - if (ranges.length < 2) { - return ranges; - } - for (let i = 1; i < ranges.length; i++) { - if (ranges[i].start.line === lastRange.end.line + 1) { - lastRange = new vscode.Range( - new vscode.Position(lastRange.start.line, 0), - new vscode.Position(ranges[i].end.line, 0) - ); - } else { - combinedRanges.push(lastRange); - lastRange = ranges[i]; - } - } - combinedRanges.push(lastRange); - return combinedRanges; - } - - private clear(editor: vscode.TextEditor) { - editor.setDecorations(this.coverageHitDecorationType, []); - editor.setDecorations(this.coverageMissDecorationType, []); - if (configuration.alwaysShowCoverageStatusItem) { - this.updateCoverageStatusItemText(this.statusItemCoverageOffText()); - } else { - this.statusBarItem.hide(); - } - } - - private statusItemCoverageOffText(): string { - if (this.displayResults) { - return "Coverage: Unavailable"; - } else { - return "Coverage: Off"; - } - } -} diff --git a/src/coverage/TestCoverageReport.ts b/src/coverage/TestCoverageReport.ts deleted file mode 100644 index 5a7166255..000000000 --- a/src/coverage/TestCoverageReport.ts +++ /dev/null @@ -1,84 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the VSCode Swift open source project -// -// Copyright (c) 2023 the VSCode Swift project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of VSCode Swift project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import * as vscode from "vscode"; -import * as path from "path"; -import { FolderContext } from "../FolderContext"; -import { WorkspaceContext } from "../WorkspaceContext"; - -export class TestCoverageReportProvider implements vscode.Disposable { - provider: vscode.Disposable; - onDidChangeEmitter: vscode.EventEmitter; - constructor(private ctx: WorkspaceContext) { - this.onDidChangeEmitter = new vscode.EventEmitter(); - this.provider = vscode.workspace.registerTextDocumentContentProvider("swiftTestCoverage", { - provideTextDocumentContent: uri => { - const folderName = path.basename(uri.path, " coverage"); - const folder = ctx.folders.find(folder => folder.name === folderName); - if (!folder) { - return `Test coverage report for ${folderName} is unavailable`; - } - const report = this.generateMarkdownReport(folder); - return report ?? `Failed to generate test coverage report for ${folderName}`; - }, - onDidChange: this.onDidChangeEmitter.event, - }); - } - - dispose() { - this.provider.dispose(); - this.onDidChangeEmitter.dispose(); - } - - show(folder: FolderContext) { - const testCoverageUri = vscode.Uri.parse( - `swiftTestCoverage://report/${folder.name} coverage` - ); - this.onDidChangeEmitter.fire(testCoverageUri); - vscode.commands.executeCommand("markdown.showPreview", testCoverageUri).then(() => { - vscode.commands.executeCommand("markdown.preview.refresh", testCoverageUri); - }); - } - - generateMarkdownReport(folder: FolderContext): string | undefined { - const lcov = folder.lcovResults; - if (!lcov.contents) { - return undefined; - } - const header = ` -## Test coverage report for ${folder.name} - -|File|Total lines| Hit|Missed|Coverage %| -|----|----------:|---:|-----:|---------:| -`; - const files = lcov.contents.map(file => { - const filename = path.basename(file.file); - const total = file.lines.found; - const hit = file.lines.hit; - const missed = total - hit; - const percent = ((100.0 * hit) / total).toFixed(2); - const fullFilename = encodeURI(`vscode://file${file.file}`); - return `|[${filename}](${fullFilename})|${total}|${hit}|${missed}|${percent}|`; - }); - const lcovTotals = lcov.totals; - - const total = lcovTotals!.found; - const hit = lcovTotals!.hit; - const missed = total - hit; - const percent = ((100.0 * hit) / total).toFixed(2); - const totals = `\n| | | | |\n|**Totals**|${total}|${hit}|${missed}|${percent}|\n`; - - return header + files.join("\n") + totals; - } -} diff --git a/src/debugger/buildConfig.ts b/src/debugger/buildConfig.ts new file mode 100644 index 000000000..18ff627df --- /dev/null +++ b/src/debugger/buildConfig.ts @@ -0,0 +1,425 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the VSCode Swift open source project +// +// Copyright (c) 2024 the VSCode Swift project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of VSCode Swift project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import * as os from "os"; +import * as path from "path"; +import * as vscode from "vscode"; +import configuration from "../configuration"; +import { FolderContext } from "../FolderContext"; +import { BuildFlags } from "../toolchain/BuildFlags"; +import { regexEscapedString, swiftRuntimeEnv } from "../utilities/utilities"; +import { DebugAdapter } from "./debugAdapter"; +import { TargetType } from "../SwiftPackage"; +import { Version } from "../utilities/version"; +import { TestKind, TestLibrary } from "../TestExplorer/TestRunner"; + +/** + * Creates `vscode.DebugConfiguration`s for different combinations of + * testing library, test kind and platform. Use the static `swiftTestingConfig` + * and `xcTestConfig` functions to create + */ +export class TestingDebugConfigurationFactory { + public static swiftTestingConfig( + ctx: FolderContext, + fifoPipePath: string, + testKind: TestKind, + testList: string[], + expandEnvVariables = false + ): vscode.DebugConfiguration | null { + return new TestingDebugConfigurationFactory( + ctx, + fifoPipePath, + testKind, + TestLibrary.swiftTesting, + testList, + expandEnvVariables + ).build(); + } + + public static xcTestConfig( + ctx: FolderContext, + testKind: TestKind, + testList: string[], + expandEnvVariables = false + ): vscode.DebugConfiguration | null { + return new TestingDebugConfigurationFactory( + ctx, + "", + testKind, + TestLibrary.xctest, + testList, + expandEnvVariables + ).build(); + } + + private constructor( + private ctx: FolderContext, + private fifoPipePath: string, + private testKind: TestKind, + private testLibrary: TestLibrary, + private testList: string[], + private expandEnvVariables = false + ) {} + + /** + * Builds a `vscode.DebugConfiguration` for running tests based on four main criteria: + * + * - Platform + * - Toolchain + * - Test Kind (coverage, debugging) + * - Test Library (XCTest, swift-testing) + */ + private build(): vscode.DebugConfiguration | null { + if (!this.hasTestTarget) { + return null; + } + + switch (process.platform) { + case "win32": + return this.buildWindowsConfig(); + case "darwin": + return this.buildDarwinConfg(); + default: + return this.buildLinuxConfig(); + } + } + + /* eslint-disable no-case-declarations */ + private buildLinuxConfig(): vscode.DebugConfiguration | null { + if (this.testKind === TestKind.debug && this.testLibrary === TestLibrary.xctest) { + const { folder } = getFolderAndNameSuffix(this.ctx, this.expandEnvVariables); + const buildDirectory = BuildFlags.buildDirectoryFromWorkspacePath(folder, true); + return { + ...this.baseConfig, + program: path.join( + buildDirectory, + "debug", + this.ctx.swiftPackage.name + "PackageTests.xctest" + ), + args: this.testList, + env: { + ...swiftRuntimeEnv(), + ...configuration.folder(this.ctx.workspaceFolder).testEnvironmentVariables, + }, + }; + } else { + return this.buildDarwinConfg(); + } + } + + private buildDarwinConfg(): vscode.DebugConfiguration | null { + switch (this.testLibrary) { + case TestLibrary.swiftTesting: + switch (this.testKind) { + case TestKind.debug: + // In the debug case we need to build the .swift-testing executable and then + // launch it with LLDB instead of going through `swift test`. + const { folder } = getFolderAndNameSuffix( + this.ctx, + this.expandEnvVariables + ); + const buildDirectory = BuildFlags.buildDirectoryFromWorkspacePath( + folder, + true + ); + const toolchain = this.ctx.workspaceContext.toolchain; + const libraryPath = toolchain.swiftTestingLibraryPath(); + const frameworkPath = toolchain.swiftTestingFrameworkPath(); + const result = { + ...this.baseConfig, + program: path.join( + buildDirectory, + "debug", + `${this.ctx.swiftPackage.name}PackageTests.swift-testing` + ), + args: this.addTestsToArgs(this.addSwiftTestingFlagsArgs([])), + env: { + ...this.testEnv, + ...this.sanitizerRuntimeEnvironment, + DYLD_FRAMEWORK_PATH: frameworkPath, + DYLD_LIBRARY_PATH: libraryPath, + SWT_SF_SYMBOLS_ENABLED: "0", + }, + }; + return result; + default: + let args = this.addSwiftTestingFlagsArgs([ + "test", + ...(this.testKind === TestKind.coverage + ? ["--enable-code-coverage"] + : []), + ]); + + if (this.swiftVersionGreaterOrEqual(6, 0, 0)) { + args = [...args, "--disable-xctest"]; + } + + return { + ...this.baseConfig, + program: this.swiftProgramPath, + args: this.addTestsToArgs(args), + env: { + ...this.testEnv, + ...this.sanitizerRuntimeEnvironment, + SWT_SF_SYMBOLS_ENABLED: "0", + }, + // For coverage we need to rebuild so do the build/test all in one step, + // otherwise we do a build, then test, to give better progress. + preLaunchTask: + this.testKind === TestKind.coverage + ? undefined + : this.baseConfig.preLaunchTask, + }; + } + case TestLibrary.xctest: + switch (this.testKind) { + case TestKind.debug: + const xcTestPath = this.ctx.workspaceContext.toolchain.xcTestPath; + // On macOS, find the path to xctest + // and point it at the .xctest bundle from the configured build directory. + if (xcTestPath === undefined) { + return null; + } + const { folder } = getFolderAndNameSuffix( + this.ctx, + this.expandEnvVariables + ); + const buildDirectory = BuildFlags.buildDirectoryFromWorkspacePath( + folder, + true + ); + return { + ...this.baseConfig, + program: path.join(xcTestPath, "xctest"), + args: this.addXCTestExutableTestsToArgs([ + path.join( + buildDirectory, + "debug", + this.ctx.swiftPackage.name + "PackageTests.xctest" + ), + ]), + env: { + ...this.testEnv, + ...this.sanitizerRuntimeEnvironment, + SWT_SF_SYMBOLS_ENABLED: "0", + }, + }; + default: + const swiftVersion = this.ctx.workspaceContext.toolchain.swiftVersion; + if ( + swiftVersion.isLessThan(new Version(5, 7, 0)) && + swiftVersion.isGreaterThanOrEqual(new Version(5, 6, 0)) && + process.platform === "darwin" + ) { + // if debugging on macOS with Swift 5.6 we need to create a custom launch + // configuration so we can set the system architecture + return this.createDarwin56TestConfiguration(); + } + + let xcTestArgs = [ + "test", + ...(this.testKind === TestKind.coverage + ? ["--enable-code-coverage"] + : []), + ]; + if (this.swiftVersionGreaterOrEqual(6, 0, 0)) { + xcTestArgs = [ + ...xcTestArgs, + "--enable-xctest", + "--disable-experimental-swift-testing", + ]; + } + + if (this.testKind === TestKind.parallel) { + xcTestArgs = [...xcTestArgs, "--parallel"]; + } + + return { + ...this.baseConfig, + program: this.swiftProgramPath, + args: this.addTestsToArgs(xcTestArgs), + env: { + ...this.testEnv, + ...this.sanitizerRuntimeEnvironment, + SWT_SF_SYMBOLS_ENABLED: "0", + }, + // For coverage we need to rebuild so do the build/test all in one step, + // otherwise we do a build, then test, to give better progress. + preLaunchTask: + this.testKind === TestKind.coverage + ? undefined + : this.baseConfig.preLaunchTask, + }; + } + } + } + + private buildWindowsConfig(): vscode.DebugConfiguration | null { + switch (this.testLibrary) { + case TestLibrary.swiftTesting: + // TODO: This is untested until rdar://128092675 is available in a windows SDK. + return this.buildDarwinConfg(); + case TestLibrary.xctest: + return this.buildDarwinConfg(); + } + } + /* eslint-enable no-case-declarations */ + + /** + * Return custom Darwin test configuration that works with Swift 5.6 + **/ + private createDarwin56TestConfiguration(): vscode.DebugConfiguration | null { + if (this.ctx.swiftPackage.getTargets(TargetType.test).length === 0) { + return null; + } + + let testFilterArg: string; + const testList = this.testList.join(","); + if (testList.length > 0) { + testFilterArg = `-XCTest ${testList}`; + } else { + testFilterArg = ""; + } + + const { folder, nameSuffix } = getFolderAndNameSuffix(this.ctx, true); + const buildDirectory = BuildFlags.buildDirectoryFromWorkspacePath(folder, true); + // On macOS, find the path to xctest + // and point it at the .xctest bundle from the configured build directory. + const xctestPath = this.ctx.workspaceContext.toolchain.xcTestPath; + if (xctestPath === undefined) { + return null; + } + let arch: string; + switch (os.arch()) { + case "x64": + arch = "x86_64"; + break; + case "arm64": + arch = "arm64e"; + break; + default: + return null; + } + const sanitizer = this.ctx.workspaceContext.toolchain.sanitizer(configuration.sanitizer); + const envCommands = Object.entries({ + ...swiftRuntimeEnv(), + ...configuration.folder(this.ctx.workspaceFolder).testEnvironmentVariables, + ...sanitizer?.runtimeEnvironment, + }).map(([key, value]) => `settings set target.env-vars ${key}="${value}"`); + + return { + type: DebugAdapter.adapterName, + request: "custom", + sourceLanguages: ["swift"], + name: `Test ${this.ctx.swiftPackage.name}`, + targetCreateCommands: [`file -a ${arch} ${xctestPath}/xctest`], + processCreateCommands: [ + ...envCommands, + `process launch -w ${folder} -- ${testFilterArg} ${buildDirectory}/debug/${this.ctx.swiftPackage.name}PackageTests.xctest`, + ], + preLaunchTask: `swift: Build All${nameSuffix}`, + }; + } + + private addSwiftTestingFlagsArgs(args: string[]): string[] { + return [ + ...args, + "--enable-experimental-swift-testing", + "--experimental-event-stream-version", + "0", + "--experimental-event-stream-output", + this.fifoPipePath, + ]; + } + + private addTestsToArgs(args: string[]): string[] { + return [...args, ...this.testList.flatMap(arg => ["--filter", regexEscapedString(arg)])]; + } + + private addXCTestExutableTestsToArgs(args: string[]): string[] { + return [...this.testList.flatMap(arg => ["-XCTest", arg]), ...args]; + } + + private swiftVersionGreaterOrEqual(major: number, minor: number, patch: number): boolean { + return this.ctx.workspaceContext.swiftVersion.isGreaterThanOrEqual( + new Version(major, minor, patch) + ); + } + + private get swiftProgramPath(): string { + return this.ctx.workspaceContext.toolchain.getToolchainExecutable("swift"); + } + + private get buildDirectory(): string { + const { folder } = getFolderAndNameSuffix(this.ctx, this.expandEnvVariables); + return BuildFlags.buildDirectoryFromWorkspacePath(folder, true); + } + + private get xcTestOutputPath(): string { + return path.join( + this.buildDirectory, + "debug", + this.ctx.swiftPackage.name + "PackageTests.xctest" + ); + } + + private get sanitizerRuntimeEnvironment() { + return this.ctx.workspaceContext.toolchain.sanitizer(configuration.sanitizer) + ?.runtimeEnvironment; + } + + private get testEnv() { + return { + ...swiftRuntimeEnv(), + ...configuration.folder(this.ctx.workspaceFolder).testEnvironmentVariables, + }; + } + + private get baseConfig() { + const { folder, nameSuffix } = getFolderAndNameSuffix(this.ctx, this.expandEnvVariables); + return { + type: DebugAdapter.adapterName, + request: "launch", + sourceLanguages: ["swift"], + name: `Test ${this.ctx.swiftPackage.name}`, + cwd: folder, + args: [], + preLaunchTask: `swift: Build All${nameSuffix}`, + terminal: "console", + }; + } + + private get hasTestTarget(): boolean { + return this.ctx.swiftPackage.getTargets(TargetType.test).length > 0; + } +} + +export function getFolderAndNameSuffix( + ctx: FolderContext, + expandEnvVariables = false +): { folder: string; nameSuffix: string } { + const workspaceFolder = expandEnvVariables + ? ctx.workspaceFolder.uri.fsPath + : `\${workspaceFolder:${ctx.workspaceFolder.name}}`; + let folder: string; + let nameSuffix: string; + if (ctx.relativePath.length === 0) { + folder = workspaceFolder; + nameSuffix = ""; + } else { + folder = path.join(workspaceFolder, ctx.relativePath); + nameSuffix = ` (${ctx.relativePath})`; + } + return { folder: folder, nameSuffix: nameSuffix }; +} diff --git a/src/debugger/launch.ts b/src/debugger/launch.ts index e30271ec6..42fb0e829 100644 --- a/src/debugger/launch.ts +++ b/src/debugger/launch.ts @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -import * as os from "os"; import * as path from "path"; import * as vscode from "vscode"; import configuration from "../configuration"; @@ -20,7 +19,7 @@ import { FolderContext } from "../FolderContext"; import { BuildFlags } from "../toolchain/BuildFlags"; import { stringArrayInEnglish, swiftLibraryPathKey, swiftRuntimeEnv } from "../utilities/utilities"; import { DebugAdapter } from "./debugAdapter"; -import { TargetType } from "../SwiftPackage"; +import { getFolderAndNameSuffix } from "./buildConfig"; /** * Edit launch.json based on contents of Swift Package. @@ -169,263 +168,6 @@ export function createSnippetConfiguration( }; } -export function createSwiftTestConfiguration( - ctx: FolderContext, - fifoPipePath: string, - expandEnvVariables = false -): vscode.DebugConfiguration | null { - return createDebugConfiguration(ctx, fifoPipePath, expandEnvVariables, "swift-testing"); -} - -/** - * Return array of DebugConfigurations for tests based on what is in Package.swift - * @param ctx Folder context - * @param fullPath should we return configuration with full paths instead of environment vars - * @returns debug configuration - */ -export function createXCTestConfiguration( - ctx: FolderContext, - expandEnvVariables = false -): vscode.DebugConfiguration | null { - return createDebugConfiguration(ctx, "", expandEnvVariables, "XCTest"); -} - -function createDebugConfiguration( - ctx: FolderContext, - fifoPipePath: string, - expandEnvVariables = false, - type: "XCTest" | "swift-testing" -): vscode.DebugConfiguration | null { - if (ctx.swiftPackage.getTargets(TargetType.test).length === 0) { - return null; - } - - const testEnv = { - ...swiftRuntimeEnv(), - ...configuration.folder(ctx.workspaceFolder).testEnvironmentVariables, - }; - const { folder, nameSuffix } = getFolderAndNameSuffix(ctx, expandEnvVariables); - const buildDirectory = BuildFlags.buildDirectoryFromWorkspacePath(folder, true); - - const baseConfig = { - type: DebugAdapter.adapterName, - request: "launch", - sourceLanguages: ["swift"], - name: `Test ${ctx.swiftPackage.name}`, - cwd: folder, - preLaunchTask: `swift: Build All${nameSuffix}`, - }; - - let programPath; - let args: string[] = []; - let preRunCommands: string[] | undefined; - let env: object = {}; - - const swiftFolderPath = ctx.workspaceContext.toolchain.swiftFolderPath; - if (swiftFolderPath === undefined) { - return null; - } - - const xcTestPath = ctx.workspaceContext.toolchain.xcTestPath; - const runtimePath = ctx.workspaceContext.toolchain.runtimePath; - const sdkroot = configuration.sdk === "" ? process.env.SDKROOT : configuration.sdk; - const libraryPath = ctx.workspaceContext.toolchain.swiftTestingLibraryPath(); - const frameworkPath = ctx.workspaceContext.toolchain.swiftTestingFrameworkPath(); - const sanitizer = ctx.workspaceContext.toolchain.sanitizer(configuration.sanitizer); - - switch (process.platform) { - case "darwin": - switch (type) { - case "swift-testing": - programPath = path.join( - buildDirectory, - "debug", - `${ctx.swiftPackage.name}PackageTests.swift-testing` - ); - args = [ - "--experimental-event-stream-version", - "0", - "--experimental-event-stream-output", - fifoPipePath, - ]; - env = { - ...testEnv, - ...sanitizer?.runtimeEnvironment, - DYLD_FRAMEWORK_PATH: frameworkPath, - DYLD_LIBRARY_PATH: libraryPath, - SWT_SF_SYMBOLS_ENABLED: "0", - }; - break; - case "XCTest": - // On macOS, find the path to xctest - // and point it at the .xctest bundle from the configured build directory. - if (xcTestPath === undefined) { - return null; - } - - programPath = path.join(xcTestPath, "xctest"); - args = [ - path.join( - buildDirectory, - "debug", - ctx.swiftPackage.name + "PackageTests.xctest" - ), - ]; - env = { ...testEnv, ...sanitizer?.runtimeEnvironment }; - break; - } - break; - case "win32": - switch (type) { - case "swift-testing": - // On Windows, add XCTest.dll to the Path - // and run the .xctest executable from the .build directory. - if (xcTestPath === undefined) { - return null; - } - if (xcTestPath !== runtimePath) { - testEnv.Path = `${xcTestPath};${testEnv.Path ?? process.env.Path}`; - } - if (sdkroot === undefined) { - return null; - } - if ( - configuration.debugger.useDebugAdapterFromToolchain || - vscode.workspace.getConfiguration("lldb")?.get("library") - ) { - preRunCommands = [`settings set target.sdk-path ${sdkroot}`]; - } - - programPath = path.join( - buildDirectory, - "debug", - `${ctx.swiftPackage.name}PackageTests.swift-testing` - ); - args = [ - "--experimental-event-stream-version", - "0", - "--experimental-event-stream-output", - fifoPipePath, - ]; - env = testEnv; - break; - case "XCTest": - // On Windows, add XCTest.dll to the Path - // and run the .xctest executable from the .build directory. - if (xcTestPath === undefined || sdkroot === undefined) { - return null; - } - if (xcTestPath !== runtimePath) { - testEnv.Path = `${xcTestPath};${testEnv.Path ?? process.env.Path}`; - } - - if ( - configuration.debugger.useDebugAdapterFromToolchain || - vscode.workspace.getConfiguration("lldb")?.get("library") - ) { - preRunCommands = [`settings set target.sdk-path ${sdkroot}`]; - } - - programPath = path.join( - buildDirectory, - "debug", - ctx.swiftPackage.name + "PackageTests.xctest" - ); - env = testEnv; - break; - } - break; - default: - switch (type) { - case "swift-testing": - // On Linux, just run the .swift-testing executable from the configured build directory. - programPath = path.join( - buildDirectory, - "debug", - `${ctx.swiftPackage.name}PackageTests.swift-testing` - ); - args = [ - "--experimental-event-stream-version", - "0", - "--experimental-event-stream-output", - fifoPipePath, - ]; - env = { - ...testEnv, - SWT_SF_SYMBOLS_ENABLED: "0", - }; - break; - case "XCTest": - programPath = path.join( - buildDirectory, - "debug", - ctx.swiftPackage.name + "PackageTests.xctest" - ); - env = testEnv; - } - } - - return { - ...baseConfig, - program: programPath, - args: args, - env: env, - preRunCommands: preRunCommands, - }; -} - -/** Return custom Darwin test configuration that works with Swift 5.6 */ -export function createDarwinTestConfiguration( - ctx: FolderContext, - args: string -): vscode.DebugConfiguration | null { - if (ctx.swiftPackage.getTargets(TargetType.test).length === 0) { - return null; - } - if (process.platform !== "darwin") { - return null; - } - - const { folder, nameSuffix } = getFolderAndNameSuffix(ctx, true); - const buildDirectory = BuildFlags.buildDirectoryFromWorkspacePath(folder, true); - // On macOS, find the path to xctest - // and point it at the .xctest bundle from the configured build directory. - const xctestPath = ctx.workspaceContext.toolchain.xcTestPath; - if (xctestPath === undefined) { - return null; - } - let arch: string; - switch (os.arch()) { - case "x64": - arch = "x86_64"; - break; - case "arm64": - arch = "arm64e"; - break; - default: - return null; - } - const sanitizer = ctx.workspaceContext.toolchain.sanitizer(configuration.sanitizer); - const envCommands = Object.entries({ - ...swiftRuntimeEnv(), - ...configuration.folder(ctx.workspaceFolder).testEnvironmentVariables, - ...sanitizer?.runtimeEnvironment, - }).map(([key, value]) => `settings set target.env-vars ${key}="${value}"`); - - return { - type: DebugAdapter.adapterName, - request: "custom", - sourceLanguages: ["swift"], - name: `Test ${ctx.swiftPackage.name}`, - targetCreateCommands: [`file -a ${arch} ${xctestPath}/xctest`], - processCreateCommands: [ - ...envCommands, - `process launch -w ${folder} -- ${args} ${buildDirectory}/debug/${ctx.swiftPackage.name}PackageTests.xctest`, - ], - preLaunchTask: `swift: Build All${nameSuffix}`, - }; -} - /** * Run debugger for given configuration * @param config Debug configuration @@ -487,22 +229,3 @@ function updateConfigWithNewKeys( } }); } - -function getFolderAndNameSuffix( - ctx: FolderContext, - expandEnvVariables = false -): { folder: string; nameSuffix: string } { - const workspaceFolder = expandEnvVariables - ? ctx.workspaceFolder.uri.fsPath - : `\${workspaceFolder:${ctx.workspaceFolder.name}}`; - let folder: string; - let nameSuffix: string; - if (ctx.relativePath.length === 0) { - folder = workspaceFolder; - nameSuffix = ""; - } else { - folder = path.join(workspaceFolder, ctx.relativePath); - nameSuffix = ` (${ctx.relativePath})`; - } - return { folder: folder, nameSuffix: nameSuffix }; -} diff --git a/src/tasks/SwiftTaskProvider.ts b/src/tasks/SwiftTaskProvider.ts index f40a8a36d..04b69e9c3 100644 --- a/src/tasks/SwiftTaskProvider.ts +++ b/src/tasks/SwiftTaskProvider.ts @@ -289,7 +289,8 @@ export function createSwiftTask( args: string[], name: string, config: TaskConfig, - toolchain: SwiftToolchain + toolchain: SwiftToolchain, + cmdEnv: { [key: string]: string } = {} ): SwiftTask { const swift = toolchain.getToolchainExecutable("swift"); args = toolchain.buildFlags.withSwiftSDKFlags(args); @@ -309,7 +310,7 @@ export function createSwiftTask( } else { cwd = config.cwd.fsPath; }*/ - const env = { ...configuration.swiftEnvironmentVariables, ...swiftRuntimeEnv() }; + const env = { ...configuration.swiftEnvironmentVariables, ...swiftRuntimeEnv(), ...cmdEnv }; const presentation = config?.presentationOptions ?? {}; const task = new vscode.Task( { diff --git a/src/tasks/TaskQueue.ts b/src/tasks/TaskQueue.ts index a9d9a1e82..41d39e9aa 100644 --- a/src/tasks/TaskQueue.ts +++ b/src/tasks/TaskQueue.ts @@ -71,6 +71,7 @@ export class TaskOperation implements SwiftOperation { workspaceContext: WorkspaceContext, token?: vscode.CancellationToken ): Promise { + workspaceContext.outputChannel.log(`Exec Task: ${this.task.detail ?? this.task.name}`); return workspaceContext.tasks.executeTaskAndWait(this.task, token); } } diff --git a/src/toolchain/toolchain.ts b/src/toolchain/toolchain.ts index b1e48b0cd..b849f5532 100644 --- a/src/toolchain/toolchain.ts +++ b/src/toolchain/toolchain.ts @@ -144,6 +144,17 @@ export class SwiftToolchain { return Sanitizer.create(name, this); } + /** + * Returns true if the console output of `swift test --parallel` prints results + * to stdout with newlines or not. + */ + public get hasMultiLineParallelTestOutput(): boolean { + return ( + this.swiftVersion.isLessThanOrEqual(new Version(5, 6, 0)) || + this.swiftVersion.isGreaterThanOrEqual(new Version(6, 0, 0)) + ); + } + /** * Get active developer dir for Xcode */ diff --git a/src/utilities/tempFolder.ts b/src/utilities/tempFolder.ts index 879d70559..b0a5ff20a 100644 --- a/src/utilities/tempFolder.ts +++ b/src/utilities/tempFolder.ts @@ -16,10 +16,14 @@ import { tmpdir } from "os"; import * as path from "path"; import * as fs from "fs/promises"; import { randomString } from "./utilities"; +import { Disposable } from "vscode"; export class TemporaryFolder { private constructor(public path: string) {} + public createDisposableFileCollection(): DisposableFileCollection { + return new DisposableFileCollection(this); + } /** * Return random filename inside temporary folder * @param prefix Prefix of file @@ -89,3 +93,22 @@ export class TemporaryFolder { } } } + +export class DisposableFileCollection implements Disposable { + private files: string[] = []; + + constructor(private folder: TemporaryFolder) {} + + public file(prefix: string, extension?: string): string { + const filename = this.folder.filename(prefix, extension); + this.files.push(filename); + return filename; + } + + async dispose() { + for (const file of this.files) { + await fs.rm(file, { force: true }); + } + this.files = []; + } +} diff --git a/test/suite/testexplorer/TestExplorerIntegration.test.ts b/test/suite/testexplorer/TestExplorerIntegration.test.ts index 6a83ab5b2..ab2a3eb96 100644 --- a/test/suite/testexplorer/TestExplorerIntegration.test.ts +++ b/test/suite/testexplorer/TestExplorerIntegration.test.ts @@ -37,18 +37,22 @@ suite("Test Explorer Suite", function () { async function runTest( controller: vscode.TestController, runProfile: RunProfileName, - test: string + ...tests: string[] ): Promise { - const testItem = getTestItem(controller, test); - assert.ok(testItem); - const targetProfile = testExplorer.testRunProfiles.find( profile => profile.label === runProfile ); if (!targetProfile) { throw new Error(`Unable to find run profile named ${runProfile}`); } - const request = new vscode.TestRunRequest([testItem]); + + const testItems = tests.map(test => { + const testItem = getTestItem(controller, test); + assert.ok(testItem); + return testItem; + }); + + const request = new vscode.TestRunRequest(testItems); return ( await Promise.all([ @@ -113,96 +117,160 @@ suite("Test Explorer Suite", function () { } }); - // TODO: Add RunProfileName.coverage once https://github.com/swift-server/vscode-swift/pull/807 is merged. - [RunProfileName.run].forEach(runProfile => { - suite(runProfile, () => { - suite("swift-testing", function () { - suiteSetup(function () { - if (workspaceContext.swiftVersion.isLessThan(new Version(6, 0, 0))) { - this.skip(); - } - }); + // Do coverage last as it does a full rebuild, causing the stage after it to have to rebuild as well. + [RunProfileName.run, RunProfileName.runParallel, RunProfileName.coverage].forEach( + runProfile => { + let xcTestFailureMessage: string; - test("Runs passing test", async function () { - const testRun = await runTest( - testExplorer.controller, - runProfile, - "PackageTests.topLevelTestPassing()" - ); + beforeEach(() => { + // From 5.7 to 5.10 running with the --parallel option dumps the test results out + // to the console with no newlines, so it isn't possible to distinguish where errors + // begin and end. Consequently we can't record them, and so we manually mark them + // as passed or failed with the message from the xunit xml. + xcTestFailureMessage = + runProfile === RunProfileName.runParallel && + !workspaceContext.toolchain.hasMultiLineParallelTestOutput + ? "failed" + : "failed - oh no"; + }); - assertTestResults(testRun, { - passed: ["PackageTests.topLevelTestPassing()"], + suite(runProfile, () => { + suite("swift-testing", function () { + suiteSetup(function () { + if (workspaceContext.swiftVersion.isLessThan(new Version(6, 0, 0))) { + this.skip(); + } }); - }); - test("Runs failing test", async function () { - const testRun = await runTest( - testExplorer.controller, - runProfile, - "PackageTests.topLevelTestFailing()" - ); + test("Runs passing test", async function () { + const testRun = await runTest( + testExplorer.controller, + runProfile, + "PackageTests.topLevelTestPassing()" + ); - assertTestResults(testRun, { - failed: ["PackageTests.topLevelTestFailing()"], + assertTestResults(testRun, { + passed: ["PackageTests.topLevelTestPassing()"], + }); }); - }); - test("Runs Suite", async function () { - const testRun = await runTest( - testExplorer.controller, - runProfile, - "PackageTests.MixedSwiftTestingSuite" - ); + test("Runs failing test", async function () { + const testRun = await runTest( + testExplorer.controller, + runProfile, + "PackageTests.topLevelTestFailing()" + ); + + assertTestResults(testRun, { + failed: [ + { + test: "PackageTests.topLevelTestFailing()", + issues: ["Expectation failed: 1 == 2"], + }, + ], + }); + }); + + test("Runs Suite", async function () { + const testRun = await runTest( + testExplorer.controller, + runProfile, + "PackageTests.MixedSwiftTestingSuite" + ); + + assertTestResults(testRun, { + passed: [ + "PackageTests.MixedSwiftTestingSuite/testPassing()", + "PackageTests.MixedSwiftTestingSuite", + ], + skipped: ["PackageTests.MixedSwiftTestingSuite/testDisabled()"], + failed: [ + { + test: "PackageTests.MixedSwiftTestingSuite/testFailing()", + issues: ["Expectation failed: 1 == 2"], + }, + ], + }); + }); - assertTestResults(testRun, { - passed: [ - "PackageTests.MixedSwiftTestingSuite/testPassing()", + test("Runs All", async function () { + const testRun = await runTest( + testExplorer.controller, + runProfile, "PackageTests.MixedSwiftTestingSuite", - ], - skipped: ["PackageTests.MixedSwiftTestingSuite/testDisabled()"], - failed: ["PackageTests.MixedSwiftTestingSuite/testFailing()"], + "PackageTests.MixedXCTestSuite" + ); + + assertTestResults(testRun, { + passed: [ + "PackageTests.MixedSwiftTestingSuite/testPassing()", + "PackageTests.MixedSwiftTestingSuite", + "PackageTests.MixedXCTestSuite/testPassing", + ], + skipped: ["PackageTests.MixedSwiftTestingSuite/testDisabled()"], + failed: [ + { + test: "PackageTests.MixedSwiftTestingSuite/testFailing()", + issues: ["Expectation failed: 1 == 2"], + }, + { + test: "PackageTests.MixedXCTestSuite/testFailing", + issues: [xcTestFailureMessage], + }, + ], + }); }); }); - }); - suite("XCTests", () => { - test("Runs passing test", async function () { - const testRun = await runTest( - testExplorer.controller, - runProfile, - "PackageTests.PassingXCTestSuite/testPassing" - ); + suite("XCTests", () => { + test("Runs passing test", async function () { + const testRun = await runTest( + testExplorer.controller, + runProfile, + "PackageTests.PassingXCTestSuite/testPassing" + ); - assertTestResults(testRun, { - passed: ["PackageTests.PassingXCTestSuite/testPassing"], + assertTestResults(testRun, { + passed: ["PackageTests.PassingXCTestSuite/testPassing"], + }); }); - }); - test("Runs failing test", async function () { - const testRun = await runTest( - testExplorer.controller, - runProfile, - "PackageTests.FailingXCTestSuite/testFailing" - ); + test("Runs failing test", async function () { + const testRun = await runTest( + testExplorer.controller, + runProfile, + "PackageTests.FailingXCTestSuite/testFailing" + ); - assertTestResults(testRun, { - failed: ["PackageTests.FailingXCTestSuite/testFailing"], + assertTestResults(testRun, { + failed: [ + { + test: "PackageTests.FailingXCTestSuite/testFailing", + issues: [xcTestFailureMessage], + }, + ], + }); }); - }); - test("Runs Suite", async function () { - const testRun = await runTest( - testExplorer.controller, - runProfile, - "PackageTests.MixedXCTestSuite" - ); + test("Runs Suite", async function () { + const testRun = await runTest( + testExplorer.controller, + runProfile, + "PackageTests.MixedXCTestSuite" + ); - assertTestResults(testRun, { - passed: ["PackageTests.MixedXCTestSuite/testPassing"], - failed: ["PackageTests.MixedXCTestSuite/testFailing"], + assertTestResults(testRun, { + passed: ["PackageTests.MixedXCTestSuite/testPassing"], + failed: [ + { + test: "PackageTests.MixedXCTestSuite/testFailing", + issues: [xcTestFailureMessage], + }, + ], + }); }); }); }); - }); - }); + } + ); }); diff --git a/test/suite/testexplorer/utilities.ts b/test/suite/testexplorer/utilities.ts index 4d065b537..fa93dcf86 100644 --- a/test/suite/testexplorer/utilities.ts +++ b/test/suite/testexplorer/utilities.ts @@ -62,7 +62,10 @@ export function assertTestControllerHierarchy( export function assertTestResults( testRun: TestRunProxy, state: { - failed?: string[]; + failed?: { + test: string; + issues: string[]; + }[]; passed?: string[]; skipped?: string[]; errored?: string[]; @@ -71,7 +74,12 @@ export function assertTestResults( assert.deepEqual( { passed: testRun.runState.passed.map(({ id }) => id), - failed: testRun.runState.failed.map(({ id }) => id), + failed: testRun.runState.failed.map(({ test, message }) => ({ + test: test.id, + issues: Array.isArray(message) + ? message.map(({ message }) => message) + : [(message as vscode.TestMessage).message], + })), skipped: testRun.runState.skipped.map(({ id }) => id), errored: testRun.runState.errored.map(({ id }) => id), },