diff --git a/Sources/Testing/ExitTests/ExitCondition.swift b/Sources/Testing/ExitTests/ExitCondition.swift index 2950b18e7..d589b5367 100644 --- a/Sources/Testing/ExitTests/ExitCondition.swift +++ b/Sources/Testing/ExitTests/ExitCondition.swift @@ -15,8 +15,8 @@ private import _TestingInternals /// /// Values of this type are used to describe the conditions under which an exit /// test is expected to pass or fail by passing them to -/// ``expect(exitsWith:_:sourceLocation:performing:)`` or -/// ``require(exitsWith:_:sourceLocation:performing:)``. +/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or +/// ``require(exitsWith:observing:_:sourceLocation:performing:)``. @_spi(Experimental) #if SWT_NO_PROCESS_SPAWNING @available(*, unavailable, message: "Exit tests are not available on this platform.") diff --git a/Sources/Testing/ExitTests/ExitTest.swift b/Sources/Testing/ExitTests/ExitTest.swift index e73a59883..b3bf6ce4d 100644 --- a/Sources/Testing/ExitTests/ExitTest.swift +++ b/Sources/Testing/ExitTests/ExitTest.swift @@ -34,6 +34,41 @@ public struct ExitTest: Sendable, ~Copyable { /// The body closure of the exit test. fileprivate var body: @Sendable () async throws -> Void = {} + /// Storage for ``observedValues``. + /// + /// Key paths are not sendable because the properties they refer to may or may + /// not be, so this property needs to be `nonisolated(unsafe)`. It is safe to + /// use it in this fashion because `ExitTestArtifacts` is sendable. + fileprivate nonisolated(unsafe) var _observedValues = [PartialKeyPath]() + + /// Key paths representing results from within this exit test that should be + /// observed and returned to the caller. + /// + /// The testing library sets this property to match what was passed by the + /// developer to the `#expect(exitsWith:)` or `#require(exitsWith:)` macro. + /// If you are implementing an exit test handler, you can check the value of + /// this property to determine what information you need to preserve from your + /// child process. + /// + /// The value of this property always includes ``Result/exitCondition`` even + /// if the test author does not specify it. + /// + /// Within a child process running an exit test, the value of this property is + /// otherwise unspecified. + @_spi(ForToolsIntegrationOnly) + public var observedValues: [PartialKeyPath] { + get { + var result = _observedValues + if !result.contains(\.exitCondition) { // O(n), but n <= 3 (no Set needed) + result.append(\.exitCondition) + } + return result + } + set { + _observedValues = newValue + } + } + /// The source location of the exit test. /// /// The source location is unique to each exit test and is consistent between @@ -184,6 +219,9 @@ extension ExitTest { /// /// - Parameters: /// - expectedExitCondition: The expected exit condition. +/// - observedValues: An array of key paths representing results from within +/// the exit test that should be observed and returned by this macro. The +/// ``ExitTestArtifacts/exitCondition`` property is always returned. /// - expression: The expression, corresponding to `condition`, that is being /// evaluated (if available at compile time.) /// - comments: An array of comments describing the expectation. This array @@ -199,19 +237,21 @@ extension ExitTest { /// convention. func callExitTest( exitsWith expectedExitCondition: ExitCondition, + observing observedValues: [PartialKeyPath], expression: __Expression, comments: @autoclosure () -> [Comment], isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result { +) async -> Result { guard let configuration = Configuration.current ?? Configuration.all.first else { preconditionFailure("A test must be running on the current task to use #expect(exitsWith:).") } var result: ExitTestArtifacts do { - let exitTest = ExitTest(expectedExitCondition: expectedExitCondition, sourceLocation: sourceLocation) + var exitTest = ExitTest(expectedExitCondition: expectedExitCondition, sourceLocation: sourceLocation) + exitTest.observedValues = observedValues result = try await configuration.exitTestHandler(exitTest) #if os(Windows) @@ -276,11 +316,15 @@ extension ExitTest { /// the exit test. /// /// This handler is invoked when an exit test (i.e. a call to either - /// ``expect(exitsWith:_:sourceLocation:performing:)`` or - /// ``require(exitsWith:_:sourceLocation:performing:)``) is started. The - /// handler is responsible for initializing a new child environment (typically - /// a child process) and running the exit test identified by `sourceLocation` - /// there. The exit test's body can be found using ``ExitTest/find(at:)``. + /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(exitsWith:observing:_:sourceLocation:performing:)``) is started. + /// The handler is responsible for initializing a new child environment + /// (typically a child process) and running the exit test identified by + /// `sourceLocation` there. + /// + /// In the child environment, you can find the exit test again by calling + /// ``ExitTest/find(at:)`` and can run it by calling + /// ``ExitTest/callAsFunction()``. /// /// The parent environment should suspend until the results of the exit test /// are available or the child environment is otherwise terminated. The parent @@ -465,20 +509,43 @@ extension ExitTest { childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_SOURCE_LOCATION"] = String(decoding: json, as: UTF8.self) } - return try await withThrowingTaskGroup(of: ExitTestArtifacts?.self) { taskGroup in + typealias ResultUpdater = @Sendable (inout ExitTestArtifacts) -> Void + return try await withThrowingTaskGroup(of: ResultUpdater?.self) { taskGroup in + // Set up stdout and stderr streams. By POSIX convention, stdin/stdout + // are line-buffered by default and stderr is unbuffered by default. + // SEE: https://en.cppreference.com/w/cpp/io/c/std_streams + var stdoutReadEnd: FileHandle? + var stdoutWriteEnd: FileHandle? + if exitTest._observedValues.contains(\.standardOutputContent) { + try FileHandle.makePipe(readEnd: &stdoutReadEnd, writeEnd: &stdoutWriteEnd) + stdoutWriteEnd?.withUnsafeCFILEHandle { stdout in + _ = setvbuf(stdout, nil, _IOLBF, Int(BUFSIZ)) + } + } + var stderrReadEnd: FileHandle? + var stderrWriteEnd: FileHandle? + if exitTest._observedValues.contains(\.standardErrorContent) { + try FileHandle.makePipe(readEnd: &stderrReadEnd, writeEnd: &stderrWriteEnd) + stderrWriteEnd?.withUnsafeCFILEHandle { stderr in + _ = setvbuf(stderr, nil, _IONBF, Int(BUFSIZ)) + } + } + // Create a "back channel" pipe to handle events from the child process. - let backChannel = try FileHandle.Pipe() + var backChannelReadEnd: FileHandle! + var backChannelWriteEnd: FileHandle! + try FileHandle.makePipe(readEnd: &backChannelReadEnd, writeEnd: &backChannelWriteEnd) // Let the child process know how to find the back channel by setting a // known environment variable to the corresponding file descriptor // (HANDLE on Windows.) var backChannelEnvironmentVariable: String? #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD) - backChannelEnvironmentVariable = backChannel.writeEnd.withUnsafePOSIXFileDescriptor { fd in + backChannelEnvironmentVariable = backChannelWriteEnd.withUnsafePOSIXFileDescriptor { fd in fd.map(String.init(describing:)) } #elseif os(Windows) - backChannelEnvironmentVariable = backChannel.writeEnd.withUnsafeWindowsHANDLE { handle in + backChannelEnvironmentVariable = backChannelWriteEnd.withUnsafeWindowsHANDLE { handle in handle.flatMap { String(describing: UInt(bitPattern: $0)) } } #else @@ -489,32 +556,55 @@ extension ExitTest { } // Spawn the child process. - let processID = try withUnsafePointer(to: backChannel.writeEnd) { writeEnd in + let processID = try withUnsafePointer(to: backChannelWriteEnd) { backChannelWriteEnd in try spawnExecutable( atPath: childProcessExecutablePath, arguments: childArguments, environment: childEnvironment, - additionalFileHandles: .init(start: writeEnd, count: 1) + standardOutput: stdoutWriteEnd, + standardError: stderrWriteEnd, + additionalFileHandles: [backChannelWriteEnd] ) } // Await termination of the child process. taskGroup.addTask { let exitCondition = try await wait(for: processID) - return ExitTestArtifacts(exitCondition: exitCondition) + return { $0.exitCondition = exitCondition } + } + + // Read back the stdout and stderr streams. + if let stdoutReadEnd { + stdoutWriteEnd?.close() + taskGroup.addTask { + let standardOutputContent = try stdoutReadEnd.readToEnd() + return { $0.standardOutputContent = standardOutputContent } + } + } + if let stderrReadEnd { + stderrWriteEnd?.close() + taskGroup.addTask { + let standardErrorContent = try stderrReadEnd.readToEnd() + return { $0.standardErrorContent = standardErrorContent } + } } // Read back all data written to the back channel by the child process // and process it as a (minimal) event stream. - let readEnd = backChannel.closeWriteEnd() + backChannelWriteEnd.close() taskGroup.addTask { - Self._processRecords(fromBackChannel: readEnd) + Self._processRecords(fromBackChannel: backChannelReadEnd) return nil } - // This is a roundabout way of saying "and return the exit condition - // yielded by wait(for:)". - return try await taskGroup.compactMap { $0 }.first { _ in true }! + // Collate the various bits of the result. The exit condition .failure + // here is just a placeholder and will be replaced by the result of one + // of the tasks above. + var result = ExitTestArtifacts(exitCondition: .failure) + for try await update in taskGroup { + update?(&result) + } + return result } } } diff --git a/Sources/Testing/ExitTests/ExitTestArtifacts.swift b/Sources/Testing/ExitTests/ExitTestArtifacts.swift index 8a808262e..6696c791b 100644 --- a/Sources/Testing/ExitTests/ExitTestArtifacts.swift +++ b/Sources/Testing/ExitTests/ExitTestArtifacts.swift @@ -11,9 +11,9 @@ /// A type representing the result of an exit test after it has exited and /// returned control to the calling test function. /// -/// Both ``expect(exitsWith:_:sourceLocation:performing:)`` and -/// ``require(exitsWith:_:sourceLocation:performing:)`` return instances of -/// this type. +/// Both ``expect(exitsWith:observing:_:sourceLocation:performing:)`` and +/// ``require(exitsWith:observing:_:sourceLocation:performing:)`` return +/// instances of this type. /// /// - Warning: The name of this type is still unstable and subject to change. @_spi(Experimental) @@ -25,11 +25,64 @@ public struct ExitTestArtifacts: Sendable { /// /// When the exit test passes, the value of this property is equal to the /// value of the `expectedExitCondition` argument passed to - /// ``expect(exitsWith:_:sourceLocation:performing:)`` or to - /// ``require(exitsWith:_:sourceLocation:performing:)``. You can compare two - /// instances of ``ExitCondition`` with ``/Swift/Optional/==(_:_:)``. + /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or to + /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. You can + /// compare two instances of ``ExitCondition`` with + /// ``/Swift/Optional/==(_:_:)``. public var exitCondition: ExitCondition + /// All bytes written to the standard output stream of the exit test before + /// it exited. + /// + /// The value of this property may contain any arbitrary sequence of bytes, + /// including sequences that are not valid UTF-8 and cannot be decoded by + /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). + /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo) + /// instead. + /// + /// When checking the value of this property, keep in mind that the standard + /// output stream is globally accessible, and any code running in an exit + /// test may write to it including including the operating system and any + /// third-party dependencies you have declared in your package. Rather than + /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), + /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) + /// to check if expected output is present. + /// + /// To enable gathering output from the standard output stream during an + /// exit test, pass `\.standardOutputContent` in the `observedValues` + /// argument of ``expect(exitsWith:observing:_:sourceLocation:performing:)`` + /// or ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// + /// If you did not request standard output content when running an exit test, + /// the value of this property is the empty array. + public var standardOutputContent = [UInt8]() + + /// All bytes written to the standard error stream of the exit test before + /// it exited. + /// + /// The value of this property may contain any arbitrary sequence of bytes, + /// including sequences that are not valid UTF-8 and cannot be decoded by + /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). + /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo) + /// instead. + /// + /// When checking the value of this property, keep in mind that the standard + /// output stream is globally accessible, and any code running in an exit + /// test may write to it including including the operating system and any + /// third-party dependencies you have declared in your package. Rather than + /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)), + /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:)) + /// to check if expected output is present. + /// + /// To enable gathering output from the standard error stream during an exit + /// test, pass `\.standardErrorContent` in the `observedValues` argument of + /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or + /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. + /// + /// If you did not request standard error content when running an exit test, + /// the value of this property is the empty array. + public var standardErrorContent = [UInt8]() + @_spi(ForToolsIntegrationOnly) public init(exitCondition: ExitCondition) { self.exitCondition = exitCondition diff --git a/Sources/Testing/ExitTests/SpawnProcess.swift b/Sources/Testing/ExitTests/SpawnProcess.swift index 8a079d0cc..79099c072 100644 --- a/Sources/Testing/ExitTests/SpawnProcess.swift +++ b/Sources/Testing/ExitTests/SpawnProcess.swift @@ -29,6 +29,15 @@ typealias ProcessID = Never /// - arguments: The arguments to pass to the executable, not including the /// executable path. /// - environment: The environment block to pass to the executable. +/// - standardInput: If not `nil`, a file handle the child process should +/// inherit as its standard input stream. This file handle must be backed +/// by a file descriptor and be open for reading. +/// - standardOutput: If not `nil`, a file handle the child process should +/// inherit as its standard output stream. This file handle must be backed +/// by a file descriptor and be open for writing. +/// - standardError: If not `nil`, a file handle the child process should +/// inherit as its standard error stream. This file handle must be backed +/// by a file descriptor and be open for writing. /// - additionalFileHandles: A collection of file handles to inherit in the /// child process. /// @@ -42,7 +51,10 @@ func spawnExecutable( atPath executablePath: String, arguments: [String], environment: [String: String], - additionalFileHandles: UnsafeBufferPointer = .init(start: nil, count: 0) + standardInput: borrowing FileHandle? = nil, + standardOutput: borrowing FileHandle? = nil, + standardError: borrowing FileHandle? = nil, + additionalFileHandles: [UnsafePointer] = [] ) throws -> ProcessID { // Darwin and Linux differ in their optionality for the posix_spawn types we // use, so use this typealias to paper over the differences. @@ -88,27 +100,42 @@ func spawnExecutable( flags |= CShort(POSIX_SPAWN_SETSIGDEF) } - // Do not forward standard I/O. - _ = posix_spawn_file_actions_addopen(fileActions, STDIN_FILENO, "/dev/null", O_RDONLY, 0) - _ = posix_spawn_file_actions_addopen(fileActions, STDOUT_FILENO, "/dev/null", O_WRONLY, 0) - _ = posix_spawn_file_actions_addopen(fileActions, STDERR_FILENO, "/dev/null", O_WRONLY, 0) - + // Forward standard I/O streams and any explicitly added file handles. #if os(Linux) || os(FreeBSD) - var highestFD = CInt(0) + var highestFD = CInt(-1) #endif - for i in 0 ..< additionalFileHandles.count { - try additionalFileHandles[i].withUnsafePOSIXFileDescriptor { fd in + func inherit(_ fileHandle: borrowing FileHandle, as standardFD: CInt? = nil) throws { + try fileHandle.withUnsafePOSIXFileDescriptor { fd in guard let fd else { - throw SystemError(description: "A child process inherit a file handle without an associated file descriptor. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + throw SystemError(description: "A child process cannot inherit a file handle without an associated file descriptor. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } + if let standardFD { + _ = posix_spawn_file_actions_adddup2(fileActions, fd, standardFD) + } else { #if SWT_TARGET_OS_APPLE - _ = posix_spawn_file_actions_addinherit_np(fileActions, fd) + _ = posix_spawn_file_actions_addinherit_np(fileActions, fd) #elseif os(Linux) || os(FreeBSD) - highestFD = max(highestFD, fd) + highestFD = max(highestFD, fd) #endif + } + } + } + func inherit(_ fileHandle: borrowing FileHandle?, as standardFD: CInt? = nil) throws { + if fileHandle != nil { + try inherit(fileHandle!, as: standardFD) + } else if let standardFD { + let mode = (standardFD == STDIN_FILENO) ? O_RDONLY : O_WRONLY + _ = posix_spawn_file_actions_addopen(fileActions, standardFD, "/dev/null", mode, 0) } } + try inherit(standardInput, as: STDIN_FILENO) + try inherit(standardOutput, as: STDOUT_FILENO) + try inherit(standardError, as: STDERR_FILENO) + for additionalFileHandle in additionalFileHandles { + try inherit(additionalFileHandle.pointee) + } + #if SWT_TARGET_OS_APPLE // Close all other file descriptors open in the parent. flags |= CShort(POSIX_SPAWN_CLOEXEC_DEFAULT) @@ -152,57 +179,75 @@ func spawnExecutable( } #elseif os(Windows) return try _withStartupInfoEx(attributeCount: 1) { startupInfo in - // Forward the back channel's write end to the child process so that it can - // send information back to us. Note that we don't keep the pipe open as - // bidirectional, though we could if we find we need to in the future. - let inheritedHandlesBuffer = UnsafeMutableBufferPointer.allocate(capacity: additionalFileHandles.count) - defer { - inheritedHandlesBuffer.deallocate() - } - for i in 0 ..< additionalFileHandles.count { - try additionalFileHandles[i].withUnsafeWindowsHANDLE { handle in - guard let handle else { - throw SystemError(description: "A child process inherit a file handle without an associated Windows handle. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + func inherit(_ fileHandle: borrowing FileHandle, as outWindowsHANDLE: inout HANDLE?) throws { + try fileHandle.withUnsafeWindowsHANDLE { windowsHANDLE in + guard let windowsHANDLE else { + throw SystemError(description: "A child process cannot inherit a file handle without an associated Windows handle. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } - inheritedHandlesBuffer[i] = handle + outWindowsHANDLE = windowsHANDLE + } + } + func inherit(_ fileHandle: borrowing FileHandle?, as outWindowsHANDLE: inout HANDLE?) throws { + if fileHandle != nil { + try inherit(fileHandle!, as: &outWindowsHANDLE) + } else { + outWindowsHANDLE = nil } } - // Update the attribute list to hold the handle buffer. - _ = UpdateProcThreadAttribute( - startupInfo.pointee.lpAttributeList, - 0, - swt_PROC_THREAD_ATTRIBUTE_HANDLE_LIST(), - inheritedHandlesBuffer.baseAddress!, - SIZE_T(MemoryLayout.stride * inheritedHandlesBuffer.count), - nil, - nil - ) + // Forward standard I/O streams. + try inherit(standardInput, as: &startupInfo.pointee.StartupInfo.hStdInput) + try inherit(standardOutput, as: &startupInfo.pointee.StartupInfo.hStdOutput) + try inherit(standardError, as: &startupInfo.pointee.StartupInfo.hStdError) + startupInfo.pointee.StartupInfo.dwFlags |= STARTF_USESTDHANDLES - let commandLine = _escapeCommandLine(CollectionOfOne(executablePath) + arguments) - let environ = environment.map { "\($0.key)=\($0.value)" }.joined(separator: "\0") + "\0\0" + // Ensure standard I/O streams and any explicitly added file handles are + // inherited by the child process. + var inheritedHandles = [HANDLE?](repeating: nil, count: additionalFileHandles.count + 3) + try inherit(standardInput, as: &inheritedHandles[0]) + try inherit(standardOutput, as: &inheritedHandles[1]) + try inherit(standardError, as: &inheritedHandles[2]) + for i in 0 ..< additionalFileHandles.count { + try inherit(additionalFileHandles[i].pointee, as: &inheritedHandles[i + 3]) + } + inheritedHandles = inheritedHandles.compactMap(\.self) - return try commandLine.withCString(encodedAs: UTF16.self) { commandLine in - try environ.withCString(encodedAs: UTF16.self) { environ in - var processInfo = PROCESS_INFORMATION() + return try inheritedHandles.withUnsafeMutableBufferPointer { inheritedHandles in + _ = UpdateProcThreadAttribute( + startupInfo.pointee.lpAttributeList, + 0, + swt_PROC_THREAD_ATTRIBUTE_HANDLE_LIST(), + inheritedHandles.baseAddress!, + SIZE_T(MemoryLayout.stride * inheritedHandles.count), + nil, + nil + ) - guard CreateProcessW( - nil, - .init(mutating: commandLine), - nil, - nil, - true, // bInheritHandles - DWORD(CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT | EXTENDED_STARTUPINFO_PRESENT), - .init(mutating: environ), - nil, - startupInfo.pointer(to: \.StartupInfo)!, - &processInfo - ) else { - throw Win32Error(rawValue: GetLastError()) - } - _ = CloseHandle(processInfo.hThread) + let commandLine = _escapeCommandLine(CollectionOfOne(executablePath) + arguments) + let environ = environment.map { "\($0.key)=\($0.value)" }.joined(separator: "\0") + "\0\0" - return processInfo.hProcess! + return try commandLine.withCString(encodedAs: UTF16.self) { commandLine in + try environ.withCString(encodedAs: UTF16.self) { environ in + var processInfo = PROCESS_INFORMATION() + + guard CreateProcessW( + nil, + .init(mutating: commandLine), + nil, + nil, + true, // bInheritHandles + DWORD(CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT | EXTENDED_STARTUPINFO_PRESENT), + .init(mutating: environ), + nil, + startupInfo.pointer(to: \.StartupInfo)!, + &processInfo + ) else { + throw Win32Error(rawValue: GetLastError()) + } + _ = CloseHandle(processInfo.hThread) + + return processInfo.hProcess! + } } } } diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 35b09f5e7..27a2ba806 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -417,6 +417,9 @@ public macro require( /// /// - Parameters: /// - expectedExitCondition: The expected exit condition. +/// - observedValues: An array of key paths representing results from within +/// the exit test that should be observed and returned by this macro. The +/// ``ExitTestArtifacts/exitCondition`` property is always returned. /// - comment: A comment describing the expectation. /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. @@ -458,6 +461,35 @@ public macro require( /// its exit status against `exitCondition`. If they match, the exit test has /// passed; otherwise, it has failed and an issue is recorded. /// +/// ## Child process output +/// +/// By default, the child process is configured without a standard output or +/// standard error stream. If your test needs to review the content of either of +/// these streams, you can pass its key path in the `observedValues` argument: +/// +/// ```swift +/// let result = await #expect( +/// exitsWith: .failure, +/// observing: [\.standardOutputContent] +/// ) { +/// print("Goodbye, world!") +/// fatalError() +/// } +/// if let result { +/// #expect(result.standardOutputContent.contains(UInt8(ascii: "G"))) +/// } +/// ``` +/// +/// - Note: The content of the standard output and standard error streams may +/// contain any arbitrary sequence of bytes, including sequences that are not +/// valid UTF-8 and cannot be decoded by [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). +/// These streams are globally accessible within the child process, and any +/// code running in an exit test may write to it including the operating +/// system and any third-party dependencies you have declared in your package. +/// +/// The actual exit condition of the child process is always reported by the +/// testing library even if you do not specify it in `observedValues`. +/// /// ## Runtime constraints /// /// Exit tests cannot capture any state originating in the parent process or @@ -486,6 +518,7 @@ public macro require( @discardableResult @freestanding(expression) public macro expect( exitsWith expectedExitCondition: ExitCondition, + observing observedValues: [PartialKeyPath] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: @convention(thin) () async throws -> Void @@ -496,6 +529,9 @@ public macro require( /// /// - Parameters: /// - expectedExitCondition: The expected exit condition. +/// - observedValues: An array of key paths representing results from within +/// the exit test that should be observed and returned by this macro. The +/// ``ExitTestArtifacts/exitCondition`` property is always returned. /// - comment: A comment describing the expectation. /// - sourceLocation: The source location to which recorded expectations and /// issues should be attributed. @@ -539,6 +575,33 @@ public macro require( /// its exit status against `exitCondition`. If they match, the exit test has /// passed; otherwise, it has failed and an issue is recorded. /// +/// ## Child process output +/// +/// By default, the child process is configured without a standard output or +/// standard error stream. If your test needs to review the content of either of +/// these streams, you can pass its key path in the `observedValues` argument: +/// +/// ```swift +/// let result = try await #require( +/// exitsWith: .failure, +/// observing: [\.standardOutputContent] +/// ) { +/// print("Goodbye, world!") +/// fatalError() +/// } +/// #expect(result.standardOutputContent.contains(UInt8(ascii: "G"))) +/// ``` +/// +/// - Note: The content of the standard output and standard error streams may +/// contain any arbitrary sequence of bytes, including sequences that are not +/// valid UTF-8 and cannot be decoded by [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s). +/// These streams are globally accessible within the child process, and any +/// code running in an exit test may write to it including the operating +/// system and any third-party dependencies you have declared in your package. +/// +/// The actual exit condition of the child process is always reported by the +/// testing library even if you do not specify it in `observedValues`. +/// /// ## Runtime constraints /// /// Exit tests cannot capture any state originating in the parent process or @@ -567,6 +630,7 @@ public macro require( @discardableResult @freestanding(expression) public macro require( exitsWith expectedExitCondition: ExitCondition, + observing observedValues: [PartialKeyPath] = [], _ comment: @autoclosure () -> Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, performing expression: @convention(thin) () async throws -> Void diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 077d911b1..09a5964e8 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1142,15 +1142,17 @@ public func __checkClosureCall( @_spi(Experimental) public func __checkClosureCall( exitsWith expectedExitCondition: ExitCondition, + observing observedValues: [PartialKeyPath], performing body: @convention(thin) () -> Void, expression: __Expression, comments: @autoclosure () -> [Comment], isRequired: Bool, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation -) async -> Result { +) async -> Result { await callExitTest( exitsWith: expectedExitCondition, + observing: observedValues, expression: expression, comments: comments(), isRequired: isRequired, diff --git a/Sources/Testing/Support/Additions/ResultAdditions.swift b/Sources/Testing/Support/Additions/ResultAdditions.swift index b0101b4ac..f14f68c85 100644 --- a/Sources/Testing/Support/Additions/ResultAdditions.swift +++ b/Sources/Testing/Support/Additions/ResultAdditions.swift @@ -15,11 +15,23 @@ extension Result { /// `#require()` macros. Do not call it directly. @inlinable public func __expected() where Success == Void {} + /// Handle this instance as if it were returned from a call to `#require()`. + /// + /// - Warning: This function is used to implement the `#expect()` and + /// `#require()` macros. Do not call it directly. + @inlinable public func __required() throws -> Success { + try get() + } +} + +// MARK: - Optional success values + +extension Result { /// Handle this instance as if it were returned from a call to `#expect()`. /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. - @inlinable public func __expected() -> Success? { + @inlinable public func __expected() -> Success where Success == T? { try? get() } @@ -27,7 +39,8 @@ extension Result { /// /// - Warning: This function is used to implement the `#expect()` and /// `#require()` macros. Do not call it directly. - @inlinable public func __required() throws -> Success { - try get() + @inlinable public func __required() throws -> T where Success == T? { + // TODO: handle edge case where the value is nil (see #780) + try get()! } } diff --git a/Sources/Testing/Support/FileHandle.swift b/Sources/Testing/Support/FileHandle.swift index ac51622bd..e7b75e57a 100644 --- a/Sources/Testing/Support/FileHandle.swift +++ b/Sources/Testing/Support/FileHandle.swift @@ -147,11 +147,7 @@ struct FileHandle: ~Copyable, Sendable { #endif guard let fileHandle else { let errorCode = swt_errno() -#if os(Windows) - _close(fd) -#else - _TestingInternals.close(fd) -#endif + Self._close(fd) throw CError(rawValue: errorCode) } self.init(unsafeCFILEHandle: fileHandle, closeWhenDone: true) @@ -174,6 +170,21 @@ struct FileHandle: ~Copyable, Sendable { _closeWhenDone = true } + /// Close a file descriptor. + /// + /// - Parameters: + /// - fd: The file descriptor to close. If the value of this argument is + /// less than `0`, this function does nothing. + private static func _close(_ fd: CInt) { + if fd >= 0 { +#if os(Windows) + _TestingInternals._close(fd) +#else + _TestingInternals.close(fd) +#endif + } + } + /// Call a function and pass the underlying C file handle to it. /// /// - Parameters: @@ -435,70 +446,55 @@ extension FileHandle { // MARK: - Pipes extension FileHandle { - /// A type representing a bidirectional pipe between two file handles. - struct Pipe: ~Copyable, Sendable { - /// The end of the pipe capable of reading. - var readEnd: FileHandle - - /// The end of the pipe capable of writing. - var writeEnd: FileHandle - - /// Initialize a new anonymous pipe. - /// - /// - Throws: Any error that prevented creation of the pipe. - init() throws { - let (fdReadEnd, fdWriteEnd) = try withUnsafeTemporaryAllocation(of: CInt.self, capacity: 2) { fds in + /// Make a pipe connecting two new file handles. + /// + /// - Parameters: + /// - readEnd: On successful return, set to a file handle that can read + /// bytes written to `writeEnd`. On failure, set to `nil`. + /// - writeEnd: On successful return, set to a file handle that can write + /// bytes read by `writeEnd`. On failure, set to `nil`. + /// + /// - Throws: Any error preventing creation of the pipe or corresponding file + /// handles. If an error occurs, both `readEnd` and `writeEnd` are set to + /// `nil` to avoid an inconsistent state. + /// + /// - Bug: This function should return a tuple containing the file handles + /// instead of returning them via `inout` arguments. Swift does not support + /// tuples with move-only elements. ([104669935](rdar://104669935)) + static func makePipe(readEnd: inout FileHandle?, writeEnd: inout FileHandle?) throws { + var (fdReadEnd, fdWriteEnd) = try withUnsafeTemporaryAllocation(of: CInt.self, capacity: 2) { fds in #if os(Windows) - guard 0 == _pipe(fds.baseAddress, 0, _O_BINARY) else { - throw CError(rawValue: swt_errno()) - } -#else - guard 0 == pipe(fds.baseAddress!) else { - throw CError(rawValue: swt_errno()) - } -#endif - return (fds[0], fds[1]) + guard 0 == _pipe(fds.baseAddress, 0, _O_BINARY) else { + throw CError(rawValue: swt_errno()) } - - // NOTE: Partial initialization of a move-only type is disallowed, as is - // conditional initialization of a local move-only value, which is why - // this section looks a little awkward. - let readEnd: FileHandle - do { - readEnd = try FileHandle(unsafePOSIXFileDescriptor: fdReadEnd, mode: "rb") - } catch { -#if os(Windows) - _close(fdWriteEnd) #else - _TestingInternals.close(fdWriteEnd) -#endif - throw error + guard 0 == pipe(fds.baseAddress!) else { + throw CError(rawValue: swt_errno()) } - let writeEnd = try FileHandle(unsafePOSIXFileDescriptor: fdWriteEnd, mode: "wb") - self.readEnd = readEnd - self.writeEnd = writeEnd +#endif + return (fds[0], fds[1]) } - - /// Close the read end of this pipe. - /// - /// - Returns: The remaining open end of the pipe. - /// - /// After calling this function, the read end is closed and the write end - /// remains open. - consuming func closeReadEnd() -> FileHandle { - readEnd.close() - return writeEnd + defer { + Self._close(fdReadEnd) + Self._close(fdWriteEnd) } - /// Close the write end of this pipe. - /// - /// - Returns: The remaining open end of the pipe. - /// - /// After calling this function, the write end is closed and the read end - /// remains open. - consuming func closeWriteEnd() -> FileHandle { - writeEnd.close() - return readEnd + do { + defer { + fdReadEnd = -1 + } + try readEnd = FileHandle(unsafePOSIXFileDescriptor: fdReadEnd, mode: "rb") + defer { + fdWriteEnd = -1 + } + try writeEnd = FileHandle(unsafePOSIXFileDescriptor: fdWriteEnd, mode: "wb") + } catch { + // Don't leak file handles! Ensure we've cleared both pointers before + // returning so the state is consistent in the caller. + readEnd = nil + writeEnd = nil + + throw error } } } diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 92daafb2a..341b27d7d 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -373,6 +373,13 @@ extension ExitTestConditionMacro { guard let expectedExitConditionIndex else { fatalError("Could not find the exit condition for this exit test. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") } + let observationListIndex = arguments.firstIndex { $0.label?.tokenKind == .identifier("observing") } + if observationListIndex == nil { + arguments.insert( + Argument(label: "observing", expression: ArrayExprSyntax(expressions: [])), + at: arguments.index(after: expectedExitConditionIndex) + ) + } let trailingClosureIndex = arguments.firstIndex { $0.label?.tokenKind == _trailingClosureLabel.tokenKind } guard let trailingClosureIndex else { fatalError("Could not find the body argument to this exit test. Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 22da6691b..7fd3e4dcf 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -383,6 +383,27 @@ private import _TestingInternals } } } + + @Test("Result contains stdout/stderr") + func exitTestResultContainsStandardStreams() async throws { + var result = try await #require(exitsWith: .success, observing: [\.standardOutputContent]) { + try FileHandle.stdout.write("STANDARD OUTPUT") + try FileHandle.stderr.write(String("STANDARD ERROR".reversed())) + exit(EXIT_SUCCESS) + } + #expect(result.exitCondition === .success) + #expect(result.standardOutputContent.contains("STANDARD OUTPUT".utf8)) + #expect(result.standardErrorContent.isEmpty) + + result = try await #require(exitsWith: .success, observing: [\.standardErrorContent]) { + try FileHandle.stdout.write("STANDARD OUTPUT") + try FileHandle.stderr.write(String("STANDARD ERROR".reversed())) + exit(EXIT_SUCCESS) + } + #expect(result.exitCondition === .success) + #expect(result.standardOutputContent.isEmpty) + #expect(result.standardErrorContent.contains("STANDARD ERROR".utf8.reversed())) + } } // MARK: - Fixtures diff --git a/Tests/TestingTests/Support/FileHandleTests.swift b/Tests/TestingTests/Support/FileHandleTests.swift index 5d93a249f..c7f347357 100644 --- a/Tests/TestingTests/Support/FileHandleTests.swift +++ b/Tests/TestingTests/Support/FileHandleTests.swift @@ -152,9 +152,11 @@ struct FileHandleTests { #if !SWT_NO_PIPES @Test("Can recognize opened pipe") func isPipe() throws { - let pipe = try FileHandle.Pipe() - #expect(pipe.readEnd.isPipe as Bool) - #expect(pipe.writeEnd.isPipe as Bool) + var readEnd: FileHandle! + var writeEnd: FileHandle! + try FileHandle.makePipe(readEnd: &readEnd, writeEnd: &writeEnd) + #expect(readEnd.isPipe as Bool) + #expect(writeEnd.isPipe as Bool) } #endif @@ -162,13 +164,17 @@ struct FileHandleTests { @Test("Can close ends of a pipe") func closeEndsOfPipe() async throws { try await confirmation("File handle closed", expectedCount: 2) { closed in - var pipe1 = try FileHandle.Pipe() - pipe1.readEnd = try fileHandleForCloseMonitoring(with: closed) - _ = pipe1.closeReadEnd() + var pipe1ReadEnd: FileHandle! + var pipe1WriteEnd: FileHandle! + try FileHandle.makePipe(readEnd: &pipe1ReadEnd, writeEnd: &pipe1WriteEnd) + pipe1ReadEnd = try fileHandleForCloseMonitoring(with: closed) + pipe1ReadEnd.close() - var pipe2 = try FileHandle.Pipe() - pipe2.writeEnd = try fileHandleForCloseMonitoring(with: closed) - _ = pipe2.closeWriteEnd() + var pipe2ReadEnd: FileHandle! + var pipe2WriteEnd: FileHandle! + try FileHandle.makePipe(readEnd: &pipe2ReadEnd, writeEnd: &pipe2WriteEnd) + pipe2WriteEnd = try fileHandleForCloseMonitoring(with: closed) + pipe2WriteEnd.close() } } #endif