From 1d9ea962dde62bb4e33456a411b652246d60b5c2 Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Wed, 14 May 2025 23:11:38 -0700 Subject: [PATCH 1/6] Preliminary implementation of polling expectations --- .../Expectations/Expectation+Macro.swift | 167 +++++++++ .../ExpectationChecking+Macro.swift | 96 +++++ Sources/Testing/Polling/CallPolling.swift | 192 ++++++++++ Sources/Testing/Polling/Polling.swift | 293 +++++++++++++++ Sources/Testing/Polling/PollingBehavior.swift | 28 ++ Tests/TestingTests/PollingTests.swift | 352 ++++++++++++++++++ 6 files changed, 1128 insertions(+) create mode 100644 Sources/Testing/Polling/CallPolling.swift create mode 100644 Sources/Testing/Polling/Polling.swift create mode 100644 Sources/Testing/Polling/PollingBehavior.swift create mode 100644 Tests/TestingTests/PollingTests.swift diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index f85c7042b..5230417d4 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -572,3 +572,170 @@ public macro require( sourceLocation: SourceLocation = #_sourceLocation, performing expression: @escaping @Sendable @convention(thin) () async throws -> Void ) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") + +// MARK: - Polling Expectations + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// Use this overload of `#expect()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro expect( + until pollingBehavior: PollingBehavior, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - throws: The error the expression should throw. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// Use this overload of `#expect()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro expect( + until pollingBehavior: PollingBehavior, + throws error: E, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") +where E: Error & Equatable + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// - throws: A closure to confirm if the expression throws the expected error. +/// +/// Use this overload of `#expect()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro expect( + until pollingBehavior: PollingBehavior, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + performing: @Sendable () async throws -> Bool, + throws errorMatcher: @Sendable (any Error) async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// Use this overload of `#require()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro require( + until pollingBehavior: PollingBehavior, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "RequireMacro") + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// Use this overload of `#require()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro require( + until pollingBehavior: PollingBehavior, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> R? +) = #externalMacro(module: "TestingMacros", type: "RequireMacro") +where R: Sendable + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - throws: The error the expression should throw +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// +/// Use this overload of `#require()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro require( + until pollingBehavior: PollingBehavior, + throws error: E, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "RequireMacro") +where E: Error & Equatable + +/// Continuously check an expression until it matches the given PollingBehavior +/// +/// - Parameters: +/// - until: The desired PollingBehavior to check for. +/// - timeout: How long to run poll the expression until stopping. +/// - comment: A comment describing the expectation. +/// - sourceLocation: The source location to which the recorded expectations +/// and issues should be attributed. +/// - expression: The expression to be evaluated. +/// - throws: A closure to confirm if the expression throws the expected error. +/// +/// Use this overload of `#require()` when you wish to poll whether a value +/// changes as the result of activity in another task/queue/thread. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@freestanding(expression) public macro require( + until pollingBehavior: PollingBehavior, + timeout: Duration = .seconds(60), + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + expression: @Sendable () async throws -> Bool, + throws errorMatcher: @Sendable (any Error) async throws -> Bool +) = #externalMacro(module: "TestingMacros", type: "RequireMacro") diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 6d3093f2a..8144a6908 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1201,6 +1201,102 @@ public func __checkClosureCall( } #endif +// MARK: - Polling + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func __checkClosureCall( + until behavior: PollingBehavior, + timeout: Duration = .seconds(60), + performing closure: @escaping @Sendable () async throws -> Bool, + expression: __Expression, + comments: @autoclosure () -> [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result { + await callPolling( + behavior: behavior, + timeout: timeout, + closure: closure, + expression: expression, + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func __checkClosureCall( + until behavior: PollingBehavior, + throws error: E, + timeout: Duration = .seconds(60), + performing closure: @escaping @Sendable () async throws -> Bool, + expression: __Expression, + comments: @autoclosure () -> [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result where E: Error & Equatable { + await callPolling( + behavior: behavior, + throws: error, + timeout: timeout, + closure: closure, + expression: expression, + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func __checkClosureCall( + until behavior: PollingBehavior, + timeout: Duration = .seconds(60), + performing closure: @escaping @Sendable () async throws -> Bool, + throws errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, + expression: __Expression, + comments: @autoclosure () -> [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result { + await callPolling( + behavior: behavior, + timeout: timeout, + closure: closure, + errorMatcher: errorMatcher, + expression: expression, + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func __checkClosureCall( + until behavior: PollingBehavior, + timeout: Duration = .seconds(60), + performing closure: @escaping @Sendable () async throws -> R, + throws errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, + expression: __Expression, + comments: @autoclosure () -> [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result where R: Sendable { + await callPolling( + behavior: behavior, + timeout: timeout, + closure: closure, + errorMatcher: errorMatcher, + expression: expression, + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + // MARK: - /// Generate a description of an error that includes its type name if not diff --git a/Sources/Testing/Polling/CallPolling.swift b/Sources/Testing/Polling/CallPolling.swift new file mode 100644 index 000000000..4f751f761 --- /dev/null +++ b/Sources/Testing/Polling/CallPolling.swift @@ -0,0 +1,192 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// Poll an expression to check that it passes until the given duration +/// +/// - Parameters: +/// - behavior: The PollingBehavior to use. +/// - timeout: How long to poll for until we time out. +/// - closure: The closure to continuously evaluate. +/// - expression: The expression, corresponding to `condition`, that is being +/// evaluated (if available at compile time.) +/// - comments: An array of comments describing the expectation. This array +/// may be empty. +/// - isRequired: Whether or not the expectation is required. The value of +/// this argument does not affect whether or not an error is thrown on +/// failure. +/// - sourceLocation: The source location of the expectation. +/// +/// This function contains the implementation for `#expect(until:)` when no +/// error is expected and no value should be returned. +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +func callPolling( + behavior: PollingBehavior, + timeout: Duration, + closure: @escaping @Sendable () async throws -> Bool, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result{ + await Polling.run( + behavior: behavior, + timeout: timeout, + closure: { + do { + return try await closure() + } catch { + return false + } + }, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +/// Poll an expression to check that it passes until the given duration +/// +/// - Parameters: +/// - behavior: The PollingBehavior to use. +/// - timeout: How long to poll for until we time out. +/// - closure: The closure to continuously evaluate. +/// - expression: The expression, corresponding to `condition`, that is being +/// evaluated (if available at compile time.) +/// - comments: An array of comments describing the expectation. This array +/// may be empty. +/// - isRequired: Whether or not the expectation is required. The value of +/// this argument does not affect whether or not an error is thrown on +/// failure. +/// - sourceLocation: The source location of the expectation. +/// +/// This function contains the implementation for `#expect(until:)` when an +/// equatable error is expected and no value should be returned. +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +func callPolling( + behavior: PollingBehavior, + throws error: E, + timeout: Duration, + closure: @escaping @Sendable () async throws -> Bool, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result where E: Error & Equatable { + await Polling.run( + behavior: behavior, + timeout: timeout, + closure: { + do { + _ = try await closure() + return false + } catch let thrownError as E { + return thrownError == error + } catch { + return false + } + }, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +/// Poll an expression to check that it passes until the given duration +/// +/// - Parameters: +/// - behavior: The PollingBehavior to use. +/// - timeout: How long to poll for until we time out. +/// - closure: The closure to continuously evaluate. +/// - expression: The expression, corresponding to `condition`, that is being +/// evaluated (if available at compile time.) +/// - comments: An array of comments describing the expectation. This array +/// may be empty. +/// - isRequired: Whether or not the expectation is required. The value of +/// this argument does not affect whether or not an error is thrown on +/// failure. +/// - sourceLocation: The source location of the expectation. +/// +/// This function contains the implementation for `#expect(until:)` when an +/// error is expected and no value should be returned. +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +func callPolling( + behavior: PollingBehavior, + timeout: Duration, + closure: @escaping @Sendable () async throws -> Bool, + errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result { + await Polling.run( + behavior: behavior, + timeout: timeout, + closure: { + do { + _ = try await closure() + return false + } catch { + return (try? await errorMatcher(error)) == true + } + }, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} + +/// Poll an expression to check that it passes until the given duration +/// +/// - Parameters: +/// - behavior: The PollingBehavior to use. +/// - timeout: How long to poll for until we time out. +/// - closure: The closure to continuously evaluate. +/// - expression: The expression, corresponding to `condition`, that is being +/// evaluated (if available at compile time.) +/// - comments: An array of comments describing the expectation. This array +/// may be empty. +/// - isRequired: Whether or not the expectation is required. The value of +/// this argument does not affect whether or not an error is thrown on +/// failure. +/// - sourceLocation: The source location of the expectation. +/// +/// This function contains the implementation for `#require(until:)` when no +/// error is expected and a value should be returned. +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +func callPolling( + behavior: PollingBehavior, + timeout: Duration, + closure: @escaping @Sendable () async throws -> R?, + errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation +) async -> Result where R: Sendable { + await Polling.run( + behavior: behavior, + timeout: timeout, + closure: { + do { + return try await closure() + } catch { + return nil + } + }, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) +} diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift new file mode 100644 index 000000000..dd09e1a64 --- /dev/null +++ b/Sources/Testing/Polling/Polling.swift @@ -0,0 +1,293 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A type for managing polling +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +struct Polling { + /// Run polling for a closure that evaluates to a boolean value. + /// + /// - Parameters: + /// - behavior: The PollingBehavior to use. + /// - timeout: How long to poll for until we time out. + /// - closure: The closure to continuously evaluate. + /// - expression: The expression, corresponding to `condition`, that is being + /// evaluated (if available at compile time.) + /// - comments: An array of comments describing the expectation. This array + /// may be empty. + /// - isRequired: Whether or not the expectation is required. The value of + /// this argument does not affect whether or not an error is thrown on + /// failure. + /// - sourceLocation: The source location of the expectation. + static func run( + behavior: PollingBehavior, + timeout: Duration, + closure: @escaping @Sendable () async -> Bool, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation + ) async -> Result { + var expectation = Expectation( + evaluatedExpression: expression, + isPassing: true, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + + let result = await poll(expression: closure, behavior: behavior, timeout: timeout) + + let sourceContext = SourceContext(backtrace: nil, sourceLocation: sourceLocation) + + switch result { + case .timedOut: + expectation.isPassing = false + Issue( + kind: .expectationFailed(expectation), + comments: comments, + sourceContext: sourceContext + ).record() + case .timedOutWithoutRunning: + expectation.isPassing = false + Issue( + kind: .expectationFailed(expectation), + comments: comments, + sourceContext: sourceContext + ).record() + case .finished: + return __checkValue( + true, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + case .failed: + return __checkValue( + false, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + case .cancelled: + Issue( + kind: .system, + comments: comments, + sourceContext: sourceContext + ).record() + } + + return .failure(ExpectationFailedError(expectation: expectation)) + } + + /// Run polling for a closure that evaluates to an optional value. + /// + /// - Parameters: + /// - behavior: The PollingBehavior to use. + /// - timeout: How long to poll for until we time out. + /// - closure: The closure to continuously evaluate. + /// - expression: The expression, corresponding to `condition`, that is being + /// evaluated (if available at compile time.) + /// - comments: An array of comments describing the expectation. This array + /// may be empty. + /// - isRequired: Whether or not the expectation is required. The value of + /// this argument does not affect whether or not an error is thrown on + /// failure. + /// - sourceLocation: The source location of the expectation. + static func run( + behavior: PollingBehavior, + timeout: Duration, + closure: @escaping @Sendable () async -> R?, + expression: __Expression, + comments: [Comment], + isRequired: Bool, + sourceLocation: SourceLocation + ) async -> Result where R: Sendable { + var expectation = Expectation( + evaluatedExpression: expression, + isPassing: true, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + + let recorder = Recorder() + + let result = await poll(expression: { + if let value = await closure() { + await recorder.record(value: value) + return true + } + return false + }, behavior: behavior, timeout: timeout) + + let sourceContext = SourceContext(backtrace: nil, sourceLocation: sourceLocation) + + switch result { + case .timedOut: + expectation.isPassing = false + Issue( + kind: .expectationFailed(expectation), + comments: comments, + sourceContext: sourceContext + ).record() + case .timedOutWithoutRunning: + expectation.isPassing = false + Issue( + kind: .expectationFailed(expectation), + comments: comments, + sourceContext: sourceContext + ).record() + case .finished: + return __checkValue( + await recorder.lastValue, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + case .failed: + return __checkValue( + nil, + expression: expression, + comments: comments, + isRequired: isRequired, + sourceLocation: sourceLocation + ) + case .cancelled: + Issue( + kind: .system, + comments: comments, + sourceContext: sourceContext + ).record() + } + + return .failure(ExpectationFailedError(expectation: expectation)) + } + + /// A type to record the last value returned by a closure returning an optional + /// This is only used in the `#require(until:)` macro returning an optional. + private actor Recorder { + var lastValue: R? + + /// Record a new value to be returned + func record(value: R) { + self.lastValue = value + } + } + + /// The result of polling expressions + private enum PollResult { + /// The polling timed out, and the expression had run at least once. + case timedOut + /// The polling timed out, but the expression had not finished running in + /// that time. + case timedOutWithoutRunning + /// The expression exited early, and we will report a success status. + case finished + /// The expression returned false under PollingBehavior.passesAlways + case failed + /// The polling was cancelled before polling could finish + case cancelled + } + + /// The poll manager. + /// + /// This function contains the logic for continuously polling an expression, + /// as well as the logic for cancelling the polling once it times out. + /// + /// - Parameters: + /// - expression: An expression to continuously evaluate + /// - behavior: The polling behavior to use + /// - timeout: How long to poll for unitl the timeout triggers. + /// - Returns: The result of this polling. + private static func poll( + expression: @escaping @Sendable () async -> Bool, + behavior: PollingBehavior, + timeout: Duration + ) async -> PollResult { + let pollingProcessor = PollingProcessor(behavior: behavior) + return await withTaskGroup { taskGroup in + taskGroup.addTask { + do { + try await Task.sleep(for: timeout) + } catch {} + // Task.sleep will only throw if it's cancelled, at which point this + // taskgroup has already returned and we don't care about the value + // returned here. + return await pollingProcessor.didTimeout() + } + taskGroup.addTask { + while Task.isCancelled == false { + let expressionPassed = await expression() + if let result = await pollingProcessor.expressionFinished(result: expressionPassed) { + return result + } + } + // The expression was cancelled without having been finished. + // This should end up being reported as a timeout error, due to + // the earlier task added to this task group. + // But there's a chance that the overall task was cancelled. + // in which case, we should report that as a system error. + return PollResult.cancelled + } + + defer { taskGroup.cancelAll() } + return await taskGroup.next() ?? .timedOut + } + } + + /// A type to process events from `Polling.poll`. + private actor PollingProcessor { + let behavior: PollingBehavior + var hasRun = false + + init(behavior: PollingBehavior) { + self.behavior = behavior + } + + /// Record a timeout event from polling. + func didTimeout() -> PollResult { + if !hasRun { + return PollResult.timedOutWithoutRunning + } + switch behavior { + case .passesOnce: + return PollResult.timedOut + case .passesAlways: + return PollResult.finished + } + } + + /// Record that an expression finished running + /// + /// - Parameters: + /// - Result: Whether or not the polled expression passed or not. + /// + /// - Returns: A non-nil PollResult if polling should exit, otherwise nil. + func expressionFinished(result: Bool) -> PollResult? { + hasRun = true + + switch behavior { + case .passesOnce: + if result { + return .finished + } else { + return nil + } + case .passesAlways: + if !result { + return .failed + } else { + return nil + } + } + } + } +} diff --git a/Sources/Testing/Polling/PollingBehavior.swift b/Sources/Testing/Polling/PollingBehavior.swift new file mode 100644 index 000000000..ccfe6045b --- /dev/null +++ b/Sources/Testing/Polling/PollingBehavior.swift @@ -0,0 +1,28 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +/// A type representing the behavior to use while polling. +@_spi(Experimental) +@frozen public enum PollingBehavior { + /// Continuously evaluate the expression until the first time it returns + /// true. + /// If it does not pass once by the time the timeout is reached, then a + /// failure will be reported. + case passesOnce + + /// Continuously evaluate the expression until the first time it returns + /// false. + /// If the expression returns false, then a failure will be reported. + /// If the expression only returns true before the timeout is reached, then + /// no failure will be reported. + /// If the expression does not finish evaluating before the timeout is + /// reached, then a failure will be reported. + case passesAlways +} diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift new file mode 100644 index 000000000..79ce2a136 --- /dev/null +++ b/Tests/TestingTests/PollingTests.swift @@ -0,0 +1,352 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +@Suite("Polling Tests") +struct PollingTests { + @Suite("PollingBehavior.passesOnce") + struct PassesOnceBehavior { + let delta = Duration.seconds(6) + + @Test("Simple passing expressions") func trivialHappyPath() async { + await #expect(until: .passesOnce) { true } + + await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { + throw PollingTestSampleError.ohNo + } + + await #expect(until: .passesOnce, performing: { + throw PollingTestSampleError.secondCase + }, throws: { error in + (error as? PollingTestSampleError) == .secondCase + }) + + await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { + throw PollingTestSampleError.ohNo + } + } + + @Test("Simple failing expressions") func trivialSadPath() async { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesOnce) { false } + }.run(configuration: configuration) + } + } + + @Test("When the value changes from false to true during execution") func changingFromFail() async { + let incrementor = Incrementor() + + await #expect(until: .passesOnce) { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we really are only running the expression until + // the first time it passes. + } + + // and then we check the count just to double check. + #expect(await incrementor.count == 2) + } + + @Test("Unexpected Errors are treated as returning false") + func errorsReported() async { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesOnce) { + throw PollingTestSampleError.ohNo + } + }.run(configuration: configuration) + } + } + } + + @Suite("PollingBehavior.passesAlways") + struct PassesAlwaysBehavior { + // use a very generous delta for CI reasons. + let delta = Duration.seconds(6) + + @Test("Simple passing expressions") func trivialHappyPath() async { + await #expect(until: .passesAlways) { true } + } + + @Test("Simple failing expressions") func trivialSadPath() async { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesAlways) { false } + }.run(configuration: configuration) + } + } + + @Test("if the closures starts off as false, but would become true") + func changingFromFail() async { + let incrementor = Incrementor() + + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesAlways) { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we fail the test if it immediately returns false + } + }.run(configuration: configuration) + } + + #expect(await incrementor.count == 1) + } + + @Test("if the closure continues to pass") + func continuousCalling() async { + let incrementor = Incrementor() + + await #expect(until: .passesAlways) { + _ = await incrementor.increment() + return true + } + + #expect(await incrementor.count > 1) + } + + @Test("Unexpected Errors will automatically exit & fail") func errorsReported() async { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesAlways) { + throw PollingTestSampleError.ohNo + } + }.run(configuration: configuration) + } + } + } + + @Suite("Duration Tests", .disabled("time-sensitive")) struct DurationTests { + @Suite("PollingBehavior.passesOnce") + struct PassesOnceBehavior { + let delta = Duration.seconds(6) + + @Test("Simple passing expressions") func trivialHappyPath() async { + let duration = await Test.Clock().measure { + await #expect(until: .passesOnce) { true } + + await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { + throw PollingTestSampleError.ohNo + } + + await #expect(until: .passesOnce, performing: { + throw PollingTestSampleError.secondCase + }, throws: { error in + (error as? PollingTestSampleError) == .secondCase + }) + + await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { + throw PollingTestSampleError.ohNo + } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("Simple failing expressions") func trivialSadPath() async { + let duration = await Test.Clock().measure { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesOnce) { false } + }.run(configuration: configuration) + } + } + #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + } + + @Test("When the value changes from false to true during execution") func changingFromFail() async { + let incrementor = Incrementor() + + let duration = await Test.Clock().measure { + await #expect(until: .passesOnce) { + await incrementor.increment() == 2 + // this will pass only on the second invocation + // This checks that we really are only running the expression until + // the first time it passes. + } + } + + // and then we check the count just to double check. + #expect(await incrementor.count == 2) + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("Unexpected Errors are treated as returning false") + func errorsReported() async { + let duration = await Test.Clock().measure { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesOnce) { + throw PollingTestSampleError.ohNo + } + }.run(configuration: configuration) + } + } + #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + } + } + + @Suite("PollingBehavior.passesAlways") + struct PassesAlwaysBehavior { + // use a very generous delta for CI reasons. + let delta = Duration.seconds(6) + + @Test("Simple passing expressions") func trivialHappyPath() async { + let duration = await Test.Clock().measure { + await #expect(until: .passesAlways) { true } + } + #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + } + + @Test("Simple failing expressions") func trivialSadPath() async { + let duration = await Test.Clock().measure { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesAlways) { false } + }.run(configuration: configuration) + } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("if the closures starts off as false, but would become true") + func changingFromFail() async { + let incrementor = Incrementor() + + let duration = await Test.Clock().measure { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesAlways) { + await incrementor.increment() == 2 + } + // this will pass only on the second invocation + // This checks that we fail the test if it immediately returns false + }.run(configuration: configuration) + } + } + + #expect(await incrementor.count == 1) + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + + @Test("if the closure continues to pass") + func continuousCalling() async { + let incrementor = Incrementor() + + let duration = await Test.Clock().measure { + await #expect(until: .passesAlways) { + _ = await incrementor.increment() + return true + } + } + + #expect(await incrementor.count > 1) + #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + } + + @Test("Unexpected Errors will automatically exit & fail") func errorsReported() async { + let duration = await Test.Clock().measure { + await confirmation("Polling failed", expectedCount: 1) { failed in + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case .issueRecorded = event.kind { + failed() + } + } + await Test { + await #expect(until: .passesOnce) { + throw PollingTestSampleError.ohNo + } + }.run(configuration: configuration) + } + } + #expect(duration.isCloseTo(other: .zero, within: delta)) + } + } + } +} + +private enum PollingTestSampleError: Error { + case ohNo + case secondCase +} + +extension DurationProtocol { + fileprivate func isCloseTo(other: Self, within delta: Self) -> Bool { + var distance = self - other + if (distance < Self.zero) { + distance *= -1 + } + return distance <= delta + } +} + +private actor Incrementor { + var count = 0 + func increment() -> Int { + count += 1 + return count + } +} From edfe9d45346ab134ae2897e574990ce51a7afdd0 Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Mon, 26 May 2025 10:22:13 -0700 Subject: [PATCH 2/6] Make Polling into a function along the lines of Confirmation Additionally, make PollingBehavior an implementation detail of polling, instead of exposed publicly Removes any timeouts involved for polling, as they become increasingly unreliable as the system runs more and more tests --- .../Expectations/Expectation+Macro.swift | 167 ------ .../ExpectationChecking+Macro.swift | 96 ---- Sources/Testing/Issues/Issue.swift | 7 + Sources/Testing/Polling/CallPolling.swift | 192 ------- Sources/Testing/Polling/Polling.swift | 536 ++++++++++-------- Sources/Testing/Polling/PollingBehavior.swift | 28 - Tests/TestingTests/PollingTests.swift | 257 ++------- .../TestSupport/TestingAdditions.swift | 25 + 8 files changed, 382 insertions(+), 926 deletions(-) delete mode 100644 Sources/Testing/Polling/CallPolling.swift delete mode 100644 Sources/Testing/Polling/PollingBehavior.swift diff --git a/Sources/Testing/Expectations/Expectation+Macro.swift b/Sources/Testing/Expectations/Expectation+Macro.swift index 5230417d4..f85c7042b 100644 --- a/Sources/Testing/Expectations/Expectation+Macro.swift +++ b/Sources/Testing/Expectations/Expectation+Macro.swift @@ -572,170 +572,3 @@ public macro require( sourceLocation: SourceLocation = #_sourceLocation, performing expression: @escaping @Sendable @convention(thin) () async throws -> Void ) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro") - -// MARK: - Polling Expectations - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// -/// Use this overload of `#expect()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro expect( - until pollingBehavior: PollingBehavior, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - throws: The error the expression should throw. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// -/// Use this overload of `#expect()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro expect( - until pollingBehavior: PollingBehavior, - throws error: E, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") -where E: Error & Equatable - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// - throws: A closure to confirm if the expression throws the expected error. -/// -/// Use this overload of `#expect()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro expect( - until pollingBehavior: PollingBehavior, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - performing: @Sendable () async throws -> Bool, - throws errorMatcher: @Sendable (any Error) async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "ExpectMacro") - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// -/// Use this overload of `#require()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro require( - until pollingBehavior: PollingBehavior, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// -/// Use this overload of `#require()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro require( - until pollingBehavior: PollingBehavior, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> R? -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") -where R: Sendable - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - throws: The error the expression should throw -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// -/// Use this overload of `#require()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro require( - until pollingBehavior: PollingBehavior, - throws error: E, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") -where E: Error & Equatable - -/// Continuously check an expression until it matches the given PollingBehavior -/// -/// - Parameters: -/// - until: The desired PollingBehavior to check for. -/// - timeout: How long to run poll the expression until stopping. -/// - comment: A comment describing the expectation. -/// - sourceLocation: The source location to which the recorded expectations -/// and issues should be attributed. -/// - expression: The expression to be evaluated. -/// - throws: A closure to confirm if the expression throws the expected error. -/// -/// Use this overload of `#require()` when you wish to poll whether a value -/// changes as the result of activity in another task/queue/thread. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -@freestanding(expression) public macro require( - until pollingBehavior: PollingBehavior, - timeout: Duration = .seconds(60), - _ comment: @autoclosure () -> Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - expression: @Sendable () async throws -> Bool, - throws errorMatcher: @Sendable (any Error) async throws -> Bool -) = #externalMacro(module: "TestingMacros", type: "RequireMacro") diff --git a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift index 8144a6908..6d3093f2a 100644 --- a/Sources/Testing/Expectations/ExpectationChecking+Macro.swift +++ b/Sources/Testing/Expectations/ExpectationChecking+Macro.swift @@ -1201,102 +1201,6 @@ public func __checkClosureCall( } #endif -// MARK: - Polling - -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func __checkClosureCall( - until behavior: PollingBehavior, - timeout: Duration = .seconds(60), - performing closure: @escaping @Sendable () async throws -> Bool, - expression: __Expression, - comments: @autoclosure () -> [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result { - await callPolling( - behavior: behavior, - timeout: timeout, - closure: closure, - expression: expression, - comments: comments(), - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func __checkClosureCall( - until behavior: PollingBehavior, - throws error: E, - timeout: Duration = .seconds(60), - performing closure: @escaping @Sendable () async throws -> Bool, - expression: __Expression, - comments: @autoclosure () -> [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result where E: Error & Equatable { - await callPolling( - behavior: behavior, - throws: error, - timeout: timeout, - closure: closure, - expression: expression, - comments: comments(), - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func __checkClosureCall( - until behavior: PollingBehavior, - timeout: Duration = .seconds(60), - performing closure: @escaping @Sendable () async throws -> Bool, - throws errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, - expression: __Expression, - comments: @autoclosure () -> [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result { - await callPolling( - behavior: behavior, - timeout: timeout, - closure: closure, - errorMatcher: errorMatcher, - expression: expression, - comments: comments(), - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func __checkClosureCall( - until behavior: PollingBehavior, - timeout: Duration = .seconds(60), - performing closure: @escaping @Sendable () async throws -> R, - throws errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, - expression: __Expression, - comments: @autoclosure () -> [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result where R: Sendable { - await callPolling( - behavior: behavior, - timeout: timeout, - closure: closure, - errorMatcher: errorMatcher, - expression: expression, - comments: comments(), - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - // MARK: - /// Generate a description of an error that includes its type name if not diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 9a2555177..e321a1b9d 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -38,6 +38,9 @@ public struct Issue: Sendable { /// confirmed too few or too many times. indirect case confirmationMiscounted(actual: Int, expected: any RangeExpression & Sendable) + @_spi(Experimental) + case confirmationPollingFailed + /// An issue due to an `Error` being thrown by a test function and caught by /// the testing library. /// @@ -286,6 +289,8 @@ extension Issue.Kind: CustomStringConvertible { } } return "Confirmation was confirmed \(actual.counting("time")), but expected to be confirmed \(String(describingForTest: expected)) time(s)" + case .confirmationPollingFailed: + return "Confirmation polling failed" case let .errorCaught(error): return "Caught error: \(error)" case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): @@ -465,6 +470,8 @@ extension Issue.Kind { .expectationFailed(Expectation.Snapshot(snapshotting: expectation)) case .confirmationMiscounted: .unconditional + case .confirmationPollingFailed: + .unconditional case let .errorCaught(error), let .valueAttachmentFailed(error): .errorCaught(ErrorSnapshot(snapshotting: error)) case let .timeLimitExceeded(timeLimitComponents: timeLimitComponents): diff --git a/Sources/Testing/Polling/CallPolling.swift b/Sources/Testing/Polling/CallPolling.swift deleted file mode 100644 index 4f751f761..000000000 --- a/Sources/Testing/Polling/CallPolling.swift +++ /dev/null @@ -1,192 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2025 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -/// Poll an expression to check that it passes until the given duration -/// -/// - Parameters: -/// - behavior: The PollingBehavior to use. -/// - timeout: How long to poll for until we time out. -/// - closure: The closure to continuously evaluate. -/// - expression: The expression, corresponding to `condition`, that is being -/// evaluated (if available at compile time.) -/// - comments: An array of comments describing the expectation. This array -/// may be empty. -/// - isRequired: Whether or not the expectation is required. The value of -/// this argument does not affect whether or not an error is thrown on -/// failure. -/// - sourceLocation: The source location of the expectation. -/// -/// This function contains the implementation for `#expect(until:)` when no -/// error is expected and no value should be returned. -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -func callPolling( - behavior: PollingBehavior, - timeout: Duration, - closure: @escaping @Sendable () async throws -> Bool, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result{ - await Polling.run( - behavior: behavior, - timeout: timeout, - closure: { - do { - return try await closure() - } catch { - return false - } - }, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -/// Poll an expression to check that it passes until the given duration -/// -/// - Parameters: -/// - behavior: The PollingBehavior to use. -/// - timeout: How long to poll for until we time out. -/// - closure: The closure to continuously evaluate. -/// - expression: The expression, corresponding to `condition`, that is being -/// evaluated (if available at compile time.) -/// - comments: An array of comments describing the expectation. This array -/// may be empty. -/// - isRequired: Whether or not the expectation is required. The value of -/// this argument does not affect whether or not an error is thrown on -/// failure. -/// - sourceLocation: The source location of the expectation. -/// -/// This function contains the implementation for `#expect(until:)` when an -/// equatable error is expected and no value should be returned. -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -func callPolling( - behavior: PollingBehavior, - throws error: E, - timeout: Duration, - closure: @escaping @Sendable () async throws -> Bool, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result where E: Error & Equatable { - await Polling.run( - behavior: behavior, - timeout: timeout, - closure: { - do { - _ = try await closure() - return false - } catch let thrownError as E { - return thrownError == error - } catch { - return false - } - }, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -/// Poll an expression to check that it passes until the given duration -/// -/// - Parameters: -/// - behavior: The PollingBehavior to use. -/// - timeout: How long to poll for until we time out. -/// - closure: The closure to continuously evaluate. -/// - expression: The expression, corresponding to `condition`, that is being -/// evaluated (if available at compile time.) -/// - comments: An array of comments describing the expectation. This array -/// may be empty. -/// - isRequired: Whether or not the expectation is required. The value of -/// this argument does not affect whether or not an error is thrown on -/// failure. -/// - sourceLocation: The source location of the expectation. -/// -/// This function contains the implementation for `#expect(until:)` when an -/// error is expected and no value should be returned. -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -func callPolling( - behavior: PollingBehavior, - timeout: Duration, - closure: @escaping @Sendable () async throws -> Bool, - errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result { - await Polling.run( - behavior: behavior, - timeout: timeout, - closure: { - do { - _ = try await closure() - return false - } catch { - return (try? await errorMatcher(error)) == true - } - }, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} - -/// Poll an expression to check that it passes until the given duration -/// -/// - Parameters: -/// - behavior: The PollingBehavior to use. -/// - timeout: How long to poll for until we time out. -/// - closure: The closure to continuously evaluate. -/// - expression: The expression, corresponding to `condition`, that is being -/// evaluated (if available at compile time.) -/// - comments: An array of comments describing the expectation. This array -/// may be empty. -/// - isRequired: Whether or not the expectation is required. The value of -/// this argument does not affect whether or not an error is thrown on -/// failure. -/// - sourceLocation: The source location of the expectation. -/// -/// This function contains the implementation for `#require(until:)` when no -/// error is expected and a value should be returned. -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -func callPolling( - behavior: PollingBehavior, - timeout: Duration, - closure: @escaping @Sendable () async throws -> R?, - errorMatcher: @escaping @Sendable (any Error) async throws -> Bool, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation -) async -> Result where R: Sendable { - await Polling.run( - behavior: behavior, - timeout: timeout, - closure: { - do { - return try await closure() - } catch { - return nil - } - }, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) -} diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index dd09e1a64..641a4d400 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -8,273 +8,224 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // -/// A type for managing polling +/// Confirm that some expression eventually returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -struct Polling { - /// Run polling for a closure that evaluates to a boolean value. - /// - /// - Parameters: - /// - behavior: The PollingBehavior to use. - /// - timeout: How long to poll for until we time out. - /// - closure: The closure to continuously evaluate. - /// - expression: The expression, corresponding to `condition`, that is being - /// evaluated (if available at compile time.) - /// - comments: An array of comments describing the expectation. This array - /// may be empty. - /// - isRequired: Whether or not the expectation is required. The value of - /// this argument does not affect whether or not an error is thrown on - /// failure. - /// - sourceLocation: The source location of the expectation. - static func run( - behavior: PollingBehavior, - timeout: Duration, - closure: @escaping @Sendable () async -> Bool, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation - ) async -> Result { - var expectation = Expectation( - evaluatedExpression: expression, - isPassing: true, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - - let result = await poll(expression: closure, behavior: behavior, timeout: timeout) - - let sourceContext = SourceContext(backtrace: nil, sourceLocation: sourceLocation) - - switch result { - case .timedOut: - expectation.isPassing = false - Issue( - kind: .expectationFailed(expectation), - comments: comments, - sourceContext: sourceContext - ).record() - case .timedOutWithoutRunning: - expectation.isPassing = false - Issue( - kind: .expectationFailed(expectation), - comments: comments, - sourceContext: sourceContext - ).record() - case .finished: - return __checkValue( - true, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - case .failed: - return __checkValue( - false, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - case .cancelled: - Issue( - kind: .system, - comments: comments, - sourceContext: sourceContext - ).record() +public func confirmPassesEventually( + _ comment: Comment? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async { + let poller = Poller( + pollingBehavior: .passesOnce, + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await body() + } catch { + return false } - - return .failure(ExpectationFailedError(expectation: expectation)) } +} - /// Run polling for a closure that evaluates to an optional value. - /// - /// - Parameters: - /// - behavior: The PollingBehavior to use. - /// - timeout: How long to poll for until we time out. - /// - closure: The closure to continuously evaluate. - /// - expression: The expression, corresponding to `condition`, that is being - /// evaluated (if available at compile time.) - /// - comments: An array of comments describing the expectation. This array - /// may be empty. - /// - isRequired: Whether or not the expectation is required. The value of - /// this argument does not affect whether or not an error is thrown on - /// failure. - /// - sourceLocation: The source location of the expectation. - static func run( - behavior: PollingBehavior, - timeout: Duration, - closure: @escaping @Sendable () async -> R?, - expression: __Expression, - comments: [Comment], - isRequired: Bool, - sourceLocation: SourceLocation - ) async -> Result where R: Sendable { - var expectation = Expectation( - evaluatedExpression: expression, - isPassing: true, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - - let recorder = Recorder() +/// A type describing an error thrown when polling fails to return a non-nil +/// value +@_spi(Experimental) +public struct PollingFailedError: Error {} - let result = await poll(expression: { - if let value = await closure() { - await recorder.record(value: value) - return true - } +/// Confirm that some expression eventually returns a non-nil value +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Returns: The first non-nil value returned by `body`. +/// +/// - Throws: A `PollingFailedError` will be thrown if `body` never returns a +/// non-optional value +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func confirmPassesEventually( + _ comment: Comment? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> R? +) async throws -> R where R: Sendable { + let recorder = PollingRecorder() + let poller = Poller( + pollingBehavior: .passesOnce, + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await recorder.record(value: body()) + } catch { return false - }, behavior: behavior, timeout: timeout) - - let sourceContext = SourceContext(backtrace: nil, sourceLocation: sourceLocation) - - switch result { - case .timedOut: - expectation.isPassing = false - Issue( - kind: .expectationFailed(expectation), - comments: comments, - sourceContext: sourceContext - ).record() - case .timedOutWithoutRunning: - expectation.isPassing = false - Issue( - kind: .expectationFailed(expectation), - comments: comments, - sourceContext: sourceContext - ).record() - case .finished: - return __checkValue( - await recorder.lastValue, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - case .failed: - return __checkValue( - nil, - expression: expression, - comments: comments, - isRequired: isRequired, - sourceLocation: sourceLocation - ) - case .cancelled: - Issue( - kind: .system, - comments: comments, - sourceContext: sourceContext - ).record() } + } - return .failure(ExpectationFailedError(expectation: expectation)) + if let value = await recorder.lastValue { + return value } + throw PollingFailedError() +} - /// A type to record the last value returned by a closure returning an optional - /// This is only used in the `#require(until:)` macro returning an optional. - private actor Recorder { - var lastValue: R? +/// Confirm that some expression always returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, confirming that some state does not change. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func confirmAlwaysPasses( + _ comment: Comment? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async { + let poller = Poller( + pollingBehavior: .passesAlways, + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } +} - /// Record a new value to be returned - func record(value: R) { - self.lastValue = value +/// Confirm that some expression always returns a non-optional value +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Returns: The value from the last time `body` was invoked. +/// +/// - Throws: A `PollingFailedError` will be thrown if `body` ever returns a +/// non-optional value +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, confirming that some state does not change. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func confirmAlwaysPasses( + _ comment: Comment? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> R? +) async throws -> R where R: Sendable { + let recorder = PollingRecorder() + let poller = Poller( + pollingBehavior: .passesAlways, + comment: comment, + sourceLocation: sourceLocation + ) + await poller.evaluate(isolation: isolation) { + do { + return try await recorder.record(value: body()) + } catch { + return false } } - /// The result of polling expressions - private enum PollResult { - /// The polling timed out, and the expression had run at least once. - case timedOut - /// The polling timed out, but the expression had not finished running in - /// that time. - case timedOutWithoutRunning - /// The expression exited early, and we will report a success status. - case finished - /// The expression returned false under PollingBehavior.passesAlways - case failed - /// The polling was cancelled before polling could finish - case cancelled + if let value = await recorder.lastValue { + return value } + throw PollingFailedError() +} - /// The poll manager. - /// - /// This function contains the logic for continuously polling an expression, - /// as well as the logic for cancelling the polling once it times out. - /// - /// - Parameters: - /// - expression: An expression to continuously evaluate - /// - behavior: The polling behavior to use - /// - timeout: How long to poll for unitl the timeout triggers. - /// - Returns: The result of this polling. - private static func poll( - expression: @escaping @Sendable () async -> Bool, - behavior: PollingBehavior, - timeout: Duration - ) async -> PollResult { - let pollingProcessor = PollingProcessor(behavior: behavior) - return await withTaskGroup { taskGroup in - taskGroup.addTask { - do { - try await Task.sleep(for: timeout) - } catch {} - // Task.sleep will only throw if it's cancelled, at which point this - // taskgroup has already returned and we don't care about the value - // returned here. - return await pollingProcessor.didTimeout() - } - taskGroup.addTask { - while Task.isCancelled == false { - let expressionPassed = await expression() - if let result = await pollingProcessor.expressionFinished(result: expressionPassed) { - return result - } - } - // The expression was cancelled without having been finished. - // This should end up being reported as a timeout error, due to - // the earlier task added to this task group. - // But there's a chance that the overall task was cancelled. - // in which case, we should report that as a system error. - return PollResult.cancelled - } +/// A type to record the last value returned by a closure returning an optional +/// This is only used in the `confirm` polling functions evaluating an optional. +private actor PollingRecorder { + var lastValue: R? - defer { taskGroup.cancelAll() } - return await taskGroup.next() ?? .timedOut - } + /// Record a new value to be returned + func record(value: R) { + self.lastValue = value } - /// A type to process events from `Polling.poll`. - private actor PollingProcessor { - let behavior: PollingBehavior - var hasRun = false - - init(behavior: PollingBehavior) { - self.behavior = behavior + func record(value: R?) -> Bool { + if let value { + self.lastValue = value + return true + } else { + return false } + } +} - /// Record a timeout event from polling. - func didTimeout() -> PollResult { - if !hasRun { - return PollResult.timedOutWithoutRunning - } - switch behavior { - case .passesOnce: - return PollResult.timedOut - case .passesAlways: - return PollResult.finished - } - } +/// A type for managing polling +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +private struct Poller { + enum PollingBehavior { + /// Continuously evaluate the expression until the first time it returns + /// true. + /// If it does not pass once by the time the timeout is reached, then a + /// failure will be reported. + case passesOnce + + /// Continuously evaluate the expression until the first time it returns + /// false. + /// If the expression returns false, then a failure will be reported. + /// If the expression only returns true before the timeout is reached, then + /// no failure will be reported. + /// If the expression does not finish evaluating before the timeout is + /// reached, then a failure will be reported. + case passesAlways - /// Record that an expression finished running + /// Process the result of a polled expression and decide whether to continue polling. /// /// - Parameters: - /// - Result: Whether or not the polled expression passed or not. + /// - expressionResult: The result of the polled expression /// - /// - Returns: A non-nil PollResult if polling should exit, otherwise nil. - func expressionFinished(result: Bool) -> PollResult? { - hasRun = true - - switch behavior { + /// - Returns: A poll result (if polling should stop), or nil (if polling should continue) + func processFinishedExpression( + expressionResult result: Bool + ) -> PollResult? { + switch self { case .passesOnce: if result { return .finished @@ -290,4 +241,105 @@ struct Polling { } } } + + /// The result of polling expressions + enum PollResult { + /// The polling ran for the total number of iterations + case ranToCompletion + /// The expression exited early, and we will report a success status. + case finished + /// The expression returned false under PollingBehavior.passesAlways + case failed + /// The polling was cancelled before polling could finish + case cancelled + + /// Process the poll result into an issue + /// + /// - Parameters: + /// - comment: The comment to record as part of the issue + /// - sourceContext: The source context for the issue + /// - pollingBehavior: The polling behavior used. + /// - Returns: An issue if one should be recorded, otherwise nil. + func issue( + comment: Comment?, + sourceContext: SourceContext, + pollingBehavior: PollingBehavior + ) -> Issue? { + let issueKind: Issue.Kind + switch self { + case .finished, .cancelled: + return nil + case .ranToCompletion: + if case .passesAlways = pollingBehavior { + return nil + } + issueKind = .confirmationPollingFailed + case .failed: + issueKind = .confirmationPollingFailed + } + return Issue( + kind: issueKind, + comments: Array(comment), + sourceContext: sourceContext + ) + } + } + + /// The polling behavior (poll until the expression first passes, or poll + /// while the expression continues to pass) + let pollingBehavior: PollingBehavior + + /// A comment from the test author associated with the polling + let comment: Comment? + + /// The source location that asked for polling. + let sourceLocation: SourceLocation + + /// Evaluate polling, and process the result, raising an issue if necessary. + /// + /// - Parameters: + /// - body: The expression to poll + /// - Side effects: If polling fails (see `PollingBehavior`), then this will + /// record an issue. + func evaluate( + isolation: isolated (any Actor)?, + _ body: @escaping () async -> Bool + ) async { + let result = await poll( + runAmount: 1_000_000, + expression: body + ) + result.issue( + comment: comment, + sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation), + pollingBehavior: pollingBehavior + )?.record() + } + + /// This function contains the logic for continuously polling an expression, + /// as well as processing the results of that expression + /// + /// - Parameters: + /// - expression: An expression to continuously evaluate + /// - behavior: The polling behavior to use + /// - timeout: How long to poll for unitl the timeout triggers. + /// - Returns: The result of this polling. + private func poll( + runAmount: Int, + isolation: isolated (any Actor)? = #isolation, + expression: @escaping () async -> Bool + ) async -> PollResult { + for _ in 0...none } } + #expect(issues.count == 3) } @Test("When the value changes from false to true during execution") func changingFromFail() async { let incrementor = Incrementor() - await #expect(until: .passesOnce) { + await confirmPassesEventually { await incrementor.increment() == 2 // this will pass only on the second invocation // This checks that we really are only running the expression until @@ -62,75 +45,62 @@ struct PollingTests { #expect(await incrementor.count == 2) } - @Test("Unexpected Errors are treated as returning false") + @Test("Thrown errors are treated as returning false") func errorsReported() async { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } + let issues = await runTest { + await confirmPassesEventually { + throw PollingTestSampleError.ohNo } - await Test { - await #expect(until: .passesOnce) { - throw PollingTestSampleError.ohNo - } - }.run(configuration: configuration) } + #expect(issues.count == 1) } } - @Suite("PollingBehavior.passesAlways") + @Suite("confirmAlwaysPasses") struct PassesAlwaysBehavior { // use a very generous delta for CI reasons. let delta = Duration.seconds(6) @Test("Simple passing expressions") func trivialHappyPath() async { - await #expect(until: .passesAlways) { true } + await confirmAlwaysPasses { true } + } + + @Test("Returning value returns the last value from the expression") + func returnsLastValueReturned() async throws { + let incrementor = Incrementor() + let value = try await confirmAlwaysPasses { + await incrementor.increment() + } + #expect(value > 1) } @Test("Simple failing expressions") func trivialSadPath() async { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesAlways) { false } - }.run(configuration: configuration) + let issues = await runTest { + await confirmAlwaysPasses { false } + _ = try await confirmAlwaysPasses { Optional.none } } + #expect(issues.count == 3) } - @Test("if the closures starts off as false, but would become true") + @Test("if the closures starts off as true, but becomes false") func changingFromFail() async { let incrementor = Incrementor() - - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } + let issues = await runTest { + await confirmAlwaysPasses { + await incrementor.increment() == 2 + // this will pass only on the first invocation + // This checks that we fail the test if it starts failing later during + // polling } - await Test { - await #expect(until: .passesAlways) { - await incrementor.increment() == 2 - // this will pass only on the second invocation - // This checks that we fail the test if it immediately returns false - } - }.run(configuration: configuration) } - - #expect(await incrementor.count == 1) + #expect(issues.count == 1) } @Test("if the closure continues to pass") func continuousCalling() async { let incrementor = Incrementor() - await #expect(until: .passesAlways) { + await confirmAlwaysPasses { _ = await incrementor.increment() return true } @@ -138,62 +108,34 @@ struct PollingTests { #expect(await incrementor.count > 1) } - @Test("Unexpected Errors will automatically exit & fail") func errorsReported() async { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } + @Test("Thrown errors will automatically exit & fail") func errorsReported() async { + let issues = await runTest { + await confirmAlwaysPasses { + throw PollingTestSampleError.ohNo } - await Test { - await #expect(until: .passesAlways) { - throw PollingTestSampleError.ohNo - } - }.run(configuration: configuration) } + #expect(issues.count == 1) } } @Suite("Duration Tests", .disabled("time-sensitive")) struct DurationTests { - @Suite("PollingBehavior.passesOnce") + @Suite("confirmPassesEventually") struct PassesOnceBehavior { let delta = Duration.seconds(6) @Test("Simple passing expressions") func trivialHappyPath() async { let duration = await Test.Clock().measure { - await #expect(until: .passesOnce) { true } - - await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { - throw PollingTestSampleError.ohNo - } - - await #expect(until: .passesOnce, performing: { - throw PollingTestSampleError.secondCase - }, throws: { error in - (error as? PollingTestSampleError) == .secondCase - }) - - await #expect(until: .passesOnce, throws: PollingTestSampleError.ohNo) { - throw PollingTestSampleError.ohNo - } + await confirmPassesEventually { true } } #expect(duration.isCloseTo(other: .zero, within: delta)) } @Test("Simple failing expressions") func trivialSadPath() async { let duration = await Test.Clock().measure { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesOnce) { false } - }.run(configuration: configuration) + let issues = await runTest { + await confirmPassesEventually { false } } + #expect(issues.count == 1) } #expect(duration.isCloseTo(other: .seconds(60), within: delta)) } @@ -202,7 +144,7 @@ struct PollingTests { let incrementor = Incrementor() let duration = await Test.Clock().measure { - await #expect(until: .passesOnce) { + await confirmPassesEventually { await incrementor.increment() == 2 // this will pass only on the second invocation // This checks that we really are only running the expression until @@ -214,113 +156,26 @@ struct PollingTests { #expect(await incrementor.count == 2) #expect(duration.isCloseTo(other: .zero, within: delta)) } - - @Test("Unexpected Errors are treated as returning false") - func errorsReported() async { - let duration = await Test.Clock().measure { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesOnce) { - throw PollingTestSampleError.ohNo - } - }.run(configuration: configuration) - } - } - #expect(duration.isCloseTo(other: .seconds(60), within: delta)) - } } - @Suite("PollingBehavior.passesAlways") + @Suite("confirmAlwaysPasses") struct PassesAlwaysBehavior { // use a very generous delta for CI reasons. let delta = Duration.seconds(6) @Test("Simple passing expressions") func trivialHappyPath() async { let duration = await Test.Clock().measure { - await #expect(until: .passesAlways) { true } + await confirmAlwaysPasses { true } } #expect(duration.isCloseTo(other: .seconds(60), within: delta)) } @Test("Simple failing expressions") func trivialSadPath() async { let duration = await Test.Clock().measure { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesAlways) { false } - }.run(configuration: configuration) - } - } - #expect(duration.isCloseTo(other: .zero, within: delta)) - } - - @Test("if the closures starts off as false, but would become true") - func changingFromFail() async { - let incrementor = Incrementor() - - let duration = await Test.Clock().measure { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesAlways) { - await incrementor.increment() == 2 - } - // this will pass only on the second invocation - // This checks that we fail the test if it immediately returns false - }.run(configuration: configuration) - } - } - - #expect(await incrementor.count == 1) - #expect(duration.isCloseTo(other: .zero, within: delta)) - } - - @Test("if the closure continues to pass") - func continuousCalling() async { - let incrementor = Incrementor() - - let duration = await Test.Clock().measure { - await #expect(until: .passesAlways) { - _ = await incrementor.increment() - return true - } - } - - #expect(await incrementor.count > 1) - #expect(duration.isCloseTo(other: .seconds(60), within: delta)) - } - - @Test("Unexpected Errors will automatically exit & fail") func errorsReported() async { - let duration = await Test.Clock().measure { - await confirmation("Polling failed", expectedCount: 1) { failed in - var configuration = Configuration() - configuration.eventHandler = { event, _ in - if case .issueRecorded = event.kind { - failed() - } - } - await Test { - await #expect(until: .passesOnce) { - throw PollingTestSampleError.ohNo - } - }.run(configuration: configuration) + let issues = await runTest { + await confirmAlwaysPasses { false } } + #expect(issues.count == 1) } #expect(duration.isCloseTo(other: .zero, within: delta)) } diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index 4648f96af..ed2e481d2 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -94,6 +94,31 @@ func runTestFunction(named name: String, in containingType: Any.Type, configurat await runner.run() } +/// Create a ``Test`` instance for the expression and run it, returning any +/// issues recorded. +/// +/// - Parameters: +/// - expression: The test expression to run +/// +/// - Returns: The list of issues recorded. +func runTest( + testFunction: @escaping @Sendable () async throws -> Void +) async -> [Issue] { + let issues = Locked(rawValue: [Issue]()) + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + print("issue recorded: \(issue)") + issues.withLock { + $0.append(issue) + } + } + } + await Test(testFunction: testFunction).run(configuration: configuration) + return issues.rawValue +} + extension Runner { /// Initialize an instance of this type that runs the free test function /// named `testName` in the module specified in `fileID`. From 825739c6610e02976cdaea65b81ff8e514a18b9f Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Fri, 6 Jun 2025 09:00:46 -0700 Subject: [PATCH 3/6] Polling: Take in configuration arguments, add polling interval --- Sources/Testing/Polling/Polling.swift | 47 ++++++++++++++++++--------- Tests/TestingTests/PollingTests.swift | 13 ++------ 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index 641a4d400..dee08d278 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -26,12 +26,16 @@ @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmPassesEventually( _ comment: Comment? = nil, + maxPollingIterations: Int = 1000, + pollingInterval: Duration = .milliseconds(1), isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> Bool ) async { let poller = Poller( pollingBehavior: .passesOnce, + pollingIterations: maxPollingIterations, + pollingInterval: pollingInterval, comment: comment, sourceLocation: sourceLocation ) @@ -70,8 +74,11 @@ public struct PollingFailedError: Error {} /// through other forms of `confirmation`. @_spi(Experimental) @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +@discardableResult public func confirmPassesEventually( _ comment: Comment? = nil, + maxPollingIterations: Int = 1000, + pollingInterval: Duration = .milliseconds(1), isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> R? @@ -79,6 +86,8 @@ public func confirmPassesEventually( let recorder = PollingRecorder() let poller = Poller( pollingBehavior: .passesOnce, + pollingIterations: maxPollingIterations, + pollingInterval: pollingInterval, comment: comment, sourceLocation: sourceLocation ) @@ -113,12 +122,16 @@ public func confirmPassesEventually( @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmAlwaysPasses( _ comment: Comment? = nil, + maxPollingIterations: Int = 1000, + pollingInterval: Duration = .milliseconds(1), isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> Bool ) async { let poller = Poller( pollingBehavior: .passesAlways, + pollingIterations: maxPollingIterations, + pollingInterval: pollingInterval, comment: comment, sourceLocation: sourceLocation ) @@ -153,28 +166,26 @@ public func confirmAlwaysPasses( @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmAlwaysPasses( _ comment: Comment? = nil, + maxPollingIterations: Int = 1000, + pollingInterval: Duration = .milliseconds(1), isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> R? -) async throws -> R where R: Sendable { - let recorder = PollingRecorder() +) async { let poller = Poller( pollingBehavior: .passesAlways, + pollingIterations: maxPollingIterations, + pollingInterval: pollingInterval, comment: comment, sourceLocation: sourceLocation ) await poller.evaluate(isolation: isolation) { do { - return try await recorder.record(value: body()) + return try await body() != nil } catch { return false } } - - if let value = await recorder.lastValue { - return value - } - throw PollingFailedError() } /// A type to record the last value returned by a closure returning an optional @@ -289,6 +300,11 @@ private struct Poller { /// while the expression continues to pass) let pollingBehavior: PollingBehavior + // How many times to poll + let pollingIterations: Int + // Minimum waiting period between polling + let pollingInterval: Duration + /// A comment from the test author associated with the polling let comment: Comment? @@ -306,7 +322,6 @@ private struct Poller { _ body: @escaping () async -> Bool ) async { let result = await poll( - runAmount: 1_000_000, expression: body ) result.issue( @@ -325,20 +340,22 @@ private struct Poller { /// - timeout: How long to poll for unitl the timeout triggers. /// - Returns: The result of this polling. private func poll( - runAmount: Int, isolation: isolated (any Actor)? = #isolation, expression: @escaping () async -> Bool ) async -> PollResult { - for _ in 0.. 1) - } - @Test("Simple failing expressions") func trivialSadPath() async { let issues = await runTest { await confirmAlwaysPasses { false } - _ = try await confirmAlwaysPasses { Optional.none } + await confirmAlwaysPasses { Optional.none } } - #expect(issues.count == 3) + #expect(issues.count == 2) } @Test("if the closures starts off as true, but becomes false") From 151aae84a0ac3860d200fb508edbe1e0e48b4b55 Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Sat, 7 Jun 2025 19:01:43 -0700 Subject: [PATCH 4/6] Add traits for configuring polling --- Sources/Testing/Polling/Polling.swift | 156 ++++++++++++------ .../Traits/PollingConfigurationTrait.swift | 90 ++++++++++ Tests/TestingTests/PollingTests.swift | 123 +++++++++++++- .../TestSupport/TestingAdditions.swift | 28 +++- 4 files changed, 337 insertions(+), 60 deletions(-) create mode 100644 Sources/Testing/Traits/PollingConfigurationTrait.swift diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index dee08d278..cc9a79812 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -8,11 +8,29 @@ // See https://swift.org/CONTRIBUTORS.txt for Swift project authors // +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +internal let defaultPollingConfiguration = ( + maxPollingIterations: 1000, + pollingInterval: Duration.milliseconds(1) +) + /// Confirm that some expression eventually returns true /// /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. /// - isolation: The actor to which `body` is isolated, if any. /// - sourceLocation: The source location to whych any recorded issues should /// be attributed. @@ -26,16 +44,24 @@ @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmPassesEventually( _ comment: Comment? = nil, - maxPollingIterations: Int = 1000, - pollingInterval: Duration = .milliseconds(1), + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> Bool ) async { let poller = Poller( pollingBehavior: .passesOnce, - pollingIterations: maxPollingIterations, - pollingInterval: pollingInterval, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesEventuallyConfigurationTrait.pollingInterval + ), comment: comment, sourceLocation: sourceLocation ) @@ -58,6 +84,18 @@ public struct PollingFailedError: Error {} /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. /// - isolation: The actor to which `body` is isolated, if any. /// - sourceLocation: The source location to whych any recorded issues should /// be attributed. @@ -77,8 +115,8 @@ public struct PollingFailedError: Error {} @discardableResult public func confirmPassesEventually( _ comment: Comment? = nil, - maxPollingIterations: Int = 1000, - pollingInterval: Duration = .milliseconds(1), + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> R? @@ -86,8 +124,16 @@ public func confirmPassesEventually( let recorder = PollingRecorder() let poller = Poller( pollingBehavior: .passesOnce, - pollingIterations: maxPollingIterations, - pollingInterval: pollingInterval, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesEventuallyConfigurationTrait.pollingInterval + ), comment: comment, sourceLocation: sourceLocation ) @@ -110,6 +156,18 @@ public func confirmPassesEventually( /// - Parameters: /// - comment: An optional comment to apply to any issues generated by this /// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesAlwaysConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesAlwaysConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesAlwaysConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesAlwaysConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. /// - isolation: The actor to which `body` is isolated, if any. /// - sourceLocation: The source location to whych any recorded issues should /// be attributed. @@ -122,16 +180,24 @@ public func confirmPassesEventually( @available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) public func confirmAlwaysPasses( _ comment: Comment? = nil, - maxPollingIterations: Int = 1000, - pollingInterval: Duration = .milliseconds(1), + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, isolation: isolated (any Actor)? = #isolation, sourceLocation: SourceLocation = #_sourceLocation, _ body: @escaping () async throws -> Bool ) async { let poller = Poller( pollingBehavior: .passesAlways, - pollingIterations: maxPollingIterations, - pollingInterval: pollingInterval, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesAlwaysConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesAlwaysConfigurationTrait.pollingInterval + ), comment: comment, sourceLocation: sourceLocation ) @@ -144,48 +210,34 @@ public func confirmAlwaysPasses( } } -/// Confirm that some expression always returns a non-optional value +/// A helper function to de-duplicate the logic of grabbing configuration from +/// either the passed-in value (if given), the hardcoded default, and the +/// appropriate configuration trait. /// -/// - Parameters: -/// - comment: An optional comment to apply to any issues generated by this -/// function. -/// - isolation: The actor to which `body` is isolated, if any. -/// - sourceLocation: The source location to whych any recorded issues should -/// be attributed. -/// - body: The function to invoke. +/// The provided value, if non-nil is returned. Otherwise, this looks for +/// the last `TraitKind` specified, and if one exists, returns the value +/// as determined by `keyPath`. +/// If no configuration trait has been applied, then this returns the `default`. /// -/// - Returns: The value from the last time `body` was invoked. -/// -/// - Throws: A `PollingFailedError` will be thrown if `body` ever returns a -/// non-optional value -/// -/// Use polling confirmations to check that an event while a test is running in -/// complex scenarios where other forms of confirmation are insufficient. For -/// example, confirming that some state does not change. -@_spi(Experimental) -@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) -public func confirmAlwaysPasses( - _ comment: Comment? = nil, - maxPollingIterations: Int = 1000, - pollingInterval: Duration = .milliseconds(1), - isolation: isolated (any Actor)? = #isolation, - sourceLocation: SourceLocation = #_sourceLocation, - _ body: @escaping () async throws -> R? -) async { - let poller = Poller( - pollingBehavior: .passesAlways, - pollingIterations: maxPollingIterations, - pollingInterval: pollingInterval, - comment: comment, - sourceLocation: sourceLocation - ) - await poller.evaluate(isolation: isolation) { - do { - return try await body() != nil - } catch { - return false - } +/// - Parameters: +/// - providedValue: The value provided by the test author when calling +/// `confirmPassesEventually` or `confirmAlwaysPasses`. +/// - default: The harded coded default value, as defined in +/// `defaultPollingConfiguration` +/// - keyPath: The keyPath mapping from `TraitKind` to the desired value type. +private func getValueFromPollingTrait( + providedValue: Value?, + default: Value, + _ keyPath: KeyPath +) -> Value { + if let providedValue { return providedValue } + guard let test = Test.current else { return `default` } + guard let trait = test.traits.compactMap({ $0 as? TraitKind }).last else { + print("No traits of type \(TraitKind.self) found. Returning default.") + print("Traits: \(test.traits)") + return `default` } + return trait[keyPath: keyPath] } /// A type to record the last value returned by a closure returning an optional @@ -321,6 +373,8 @@ private struct Poller { isolation: isolated (any Actor)?, _ body: @escaping () async -> Bool ) async { + precondition(pollingIterations > 0) + precondition(pollingInterval > Duration.zero) let result = await poll( expression: body ) diff --git a/Sources/Testing/Traits/PollingConfigurationTrait.swift b/Sources/Testing/Traits/PollingConfigurationTrait.swift new file mode 100644 index 000000000..56e6ba968 --- /dev/null +++ b/Sources/Testing/Traits/PollingConfigurationTrait.swift @@ -0,0 +1,90 @@ +// +// PollingConfiguration.swift +// swift-testing +// +// Created by Rachel Brindle on 6/6/25. +// + +/// A trait to provide a default polling configuration to all usages of +/// ``confirmPassesEventually`` within a test or suite. +/// +/// To add this trait to a test, use the ``Trait/pollingConfirmationEventually`` +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public struct ConfirmPassesEventuallyConfigurationTrait: TestTrait, SuiteTrait { + public var maxPollingIterations: Int + public var pollingInterval: Duration + + public var isRecursive: Bool { true } + + public init(maxPollingIterations: Int?, pollingInterval: Duration?) { + self.maxPollingIterations = maxPollingIterations ?? defaultPollingConfiguration.maxPollingIterations + self.pollingInterval = pollingInterval ?? defaultPollingConfiguration.pollingInterval + } +} + +/// A trait to provide a default polling configuration to all usages of +/// ``confirmPassesAlways`` within a test or suite. +/// +/// To add this trait to a test, use the ``Trait/pollingConfirmationAlways`` +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public struct ConfirmPassesAlwaysConfigurationTrait: TestTrait, SuiteTrait { + public var maxPollingIterations: Int + public var pollingInterval: Duration + + public var isRecursive: Bool { true } + + public init(maxPollingIterations: Int?, pollingInterval: Duration?) { + self.maxPollingIterations = maxPollingIterations ?? defaultPollingConfiguration.maxPollingIterations + self.pollingInterval = pollingInterval ?? defaultPollingConfiguration.pollingInterval + } +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +extension Trait where Self == ConfirmPassesEventuallyConfigurationTrait { + /// Specifies defaults for ``confirmPassesEventually`` in the test or suite. + /// + /// - Parameters: + /// - maxPollingIterations: The maximum amount of times to attempt polling. + /// If nil, polling will be attempted up to 1000 times. + /// `maxPollingIterations` must be greater than 0. + /// - pollingInterval: The minimum amount of time to wait between polling + /// attempts. + /// If nil, polling will wait at least 1 millisecond between polling attempts. + /// `pollingInterval` must be greater than 0. + public static func confirmPassesEventuallyDefaults( + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil + ) -> Self { + ConfirmPassesEventuallyConfigurationTrait( + maxPollingIterations: maxPollingIterations, + pollingInterval: pollingInterval + ) + } +} + +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +extension Trait where Self == ConfirmPassesAlwaysConfigurationTrait { + /// Specifies defaults for ``confirmPassesAlways`` in the test or suite. + /// + /// - Parameters: + /// - maxPollingIterations: The maximum amount of times to attempt polling. + /// If nil, polling will be attempted up to 1000 times. + /// `maxPollingIterations` must be greater than 0. + /// - pollingInterval: The minimum amount of time to wait between polling + /// attempts. + /// If nil, polling will wait at least 1 millisecond between polling attempts. + /// `pollingInterval` must be greater than 0. + public static func confirmPassesAlwaysDefaults( + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil + ) -> Self { + ConfirmPassesAlwaysConfigurationTrait( + maxPollingIterations: maxPollingIterations, + pollingInterval: pollingInterval + ) + } +} diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift index cd7aed967..d6330372a 100644 --- a/Tests/TestingTests/PollingTests.swift +++ b/Tests/TestingTests/PollingTests.swift @@ -14,8 +14,6 @@ struct PollingTests { @Suite("confirmPassesEventually") struct PassesOnceBehavior { - let delta = Duration.seconds(6) - @Test("Simple passing expressions") func trivialHappyPath() async throws { await confirmPassesEventually { true } @@ -54,13 +52,76 @@ struct PollingTests { } #expect(issues.count == 1) } + + @Test("Waits up to 1000 times before failing") + func defaultPollingCount() async { + let incrementor = Incrementor() + _ = await runTest { + // this test will intentionally fail. + await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + #expect(await incrementor.count == 1000) + } + + @Suite( + "Configuration traits", + .confirmPassesEventuallyDefaults(maxPollingIterations: 100) + ) + struct WithConfigurationTraits { + @Test("When no test or callsite configuration provided, uses the suite configuration") + func testUsesSuiteConfiguration() async throws { + let incrementor = Incrementor() + var test = Test { + await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + let count = await incrementor.count + #expect(count == 100) + } + + @Test( + "When test configuration porvided, uses the test configuration", + .confirmPassesEventuallyDefaults(maxPollingIterations: 10) + ) + func testUsesTestConfigurationOverSuiteConfiguration() async { + let incrementor = Incrementor() + var test = Test { + // this test will intentionally fail. + await confirmPassesEventually(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 10) + } + + @Test( + "When callsite configuration provided, uses that", + .confirmPassesEventuallyDefaults(maxPollingIterations: 10) + ) + func testUsesCallsiteConfiguration() async { + let incrementor = Incrementor() + var test = Test { + // this test will intentionally fail. + await confirmPassesEventually(maxPollingIterations: 50, pollingInterval: .nanoseconds(1)) { + await incrementor.increment() == 0 + } + } + test.traits = Test.current?.traits ?? [] + await runTest(test: test) + #expect(await incrementor.count == 50) + } + } } @Suite("confirmAlwaysPasses") struct PassesAlwaysBehavior { - // use a very generous delta for CI reasons. - let delta = Duration.seconds(6) - @Test("Simple passing expressions") func trivialHappyPath() async { await confirmAlwaysPasses { true } } @@ -68,9 +129,8 @@ struct PollingTests { @Test("Simple failing expressions") func trivialSadPath() async { let issues = await runTest { await confirmAlwaysPasses { false } - await confirmAlwaysPasses { Optional.none } } - #expect(issues.count == 2) + #expect(issues.count == 1) } @Test("if the closures starts off as true, but becomes false") @@ -107,6 +167,55 @@ struct PollingTests { } #expect(issues.count == 1) } + + @Test("Waits up to 1000 times before passing") + func defaultPollingCount() async { + let incrementor = Incrementor() + await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 1000) + } + + @Suite( + "Configuration traits", + .confirmPassesAlwaysDefaults(maxPollingIterations: 100) + ) + struct WithConfigurationTraits { + @Test("When no test or callsite configuration provided, uses the suite configuration") + func testUsesSuiteConfiguration() async throws { + let incrementor = Incrementor() + await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + let count = await incrementor.count + #expect(count == 100) + } + + @Test( + "When test configuration porvided, uses the test configuration", + .confirmPassesAlwaysDefaults(maxPollingIterations: 10) + ) + func testUsesTestConfigurationOverSuiteConfiguration() async { + let incrementor = Incrementor() + await confirmAlwaysPasses(pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 10) + } + + @Test( + "When callsite configuration provided, uses that", + .confirmPassesAlwaysDefaults(maxPollingIterations: 10) + ) + func testUsesCallsiteConfiguration() async { + let incrementor = Incrementor() + await confirmAlwaysPasses(maxPollingIterations: 50, pollingInterval: .nanoseconds(1)) { + await incrementor.increment() != 0 + } + #expect(await incrementor.count == 50) + } + } } @Suite("Duration Tests", .disabled("time-sensitive")) struct DurationTests { diff --git a/Tests/TestingTests/TestSupport/TestingAdditions.swift b/Tests/TestingTests/TestSupport/TestingAdditions.swift index ed2e481d2..56455a710 100644 --- a/Tests/TestingTests/TestSupport/TestingAdditions.swift +++ b/Tests/TestingTests/TestSupport/TestingAdditions.swift @@ -98,9 +98,10 @@ func runTestFunction(named name: String, in containingType: Any.Type, configurat /// issues recorded. /// /// - Parameters: -/// - expression: The test expression to run +/// - testFunction: The test expression to run /// /// - Returns: The list of issues recorded. +@discardableResult func runTest( testFunction: @escaping @Sendable () async throws -> Void ) async -> [Issue] { @@ -109,7 +110,6 @@ func runTest( var configuration = Configuration() configuration.eventHandler = { event, _ in if case let .issueRecorded(issue) = event.kind { - print("issue recorded: \(issue)") issues.withLock { $0.append(issue) } @@ -119,6 +119,30 @@ func runTest( return issues.rawValue } +/// Runs the passed-in `Test`, returning any issues recorded. +/// +/// - Parameters: +/// - test: The test to run +/// +/// - Returns: The list of issues recorded. +@discardableResult +func runTest( + test: Test +) async -> [Issue] { + let issues = Locked(rawValue: [Issue]()) + + var configuration = Configuration() + configuration.eventHandler = { event, _ in + if case let .issueRecorded(issue) = event.kind { + issues.withLock { + $0.append(issue) + } + } + } + await test.run(configuration: configuration) + return issues.rawValue +} + extension Runner { /// Initialize an instance of this type that runs the free test function /// named `testName` in the module specified in `fileID`. From 2dce5112822d23590370a030941fcfed52ebf523 Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Sat, 7 Jun 2025 21:22:26 -0700 Subject: [PATCH 5/6] Use consistent naming between confirmAlwaysPasses and the related configuration trait Stop unnecessarily waiting after the last polling attempt has finished. Allow for subsequent polling configuration traits which specified nil for a value to fall back to earlier polling configuration traits before falling back to the default. --- Sources/Testing/Polling/Polling.swift | 44 ++++++++------- .../Traits/PollingConfigurationTrait.swift | 38 +++++++------ Tests/TestingTests/PollingTests.swift | 56 +++++++++++++++---- 3 files changed, 92 insertions(+), 46 deletions(-) diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index cc9a79812..ffd3978f8 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -21,11 +21,13 @@ internal let defaultPollingConfiguration = ( /// function. /// - maxPollingIterations: The maximum amount of times to attempt polling. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then /// polling will be attempted 1000 times before recording an issue. /// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. /// If nil, this uses whatever value is specified under the last /// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then @@ -86,11 +88,13 @@ public struct PollingFailedError: Error {} /// function. /// - maxPollingIterations: The maximum amount of times to attempt polling. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then /// polling will be attempted 1000 times before recording an issue. /// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. /// If nil, this uses whatever value is specified under the last /// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. /// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then @@ -158,14 +162,15 @@ public func confirmPassesEventually( /// function. /// - maxPollingIterations: The maximum amount of times to attempt polling. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesAlwaysConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmPassesAlwaysConfigurationTrait`` has been added, then +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then /// polling will be attempted 1000 times before recording an issue. /// `maxPollingIterations` must be greater than 0. -/// - pollingInterval: The minimum amount of time to wait between polling attempts. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. /// If nil, this uses whatever value is specified under the last -/// ``ConfirmPassesAlwaysConfigurationTrait`` added to the test or suite. -/// If no ``ConfirmPassesAlwaysConfigurationTrait`` has been added, then +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then /// polling will wait at least 1 millisecond between polling attempts. /// `pollingInterval` must be greater than 0. /// - isolation: The actor to which `body` is isolated, if any. @@ -191,12 +196,12 @@ public func confirmAlwaysPasses( pollingIterations: getValueFromPollingTrait( providedValue: maxPollingIterations, default: defaultPollingConfiguration.maxPollingIterations, - \ConfirmPassesAlwaysConfigurationTrait.maxPollingIterations + \ConfirmAlwaysPassesConfigurationTrait.maxPollingIterations ), pollingInterval: getValueFromPollingTrait( providedValue: pollingInterval, default: defaultPollingConfiguration.pollingInterval, - \ConfirmPassesAlwaysConfigurationTrait.pollingInterval + \ConfirmAlwaysPassesConfigurationTrait.pollingInterval ), comment: comment, sourceLocation: sourceLocation @@ -228,16 +233,13 @@ public func confirmAlwaysPasses( private func getValueFromPollingTrait( providedValue: Value?, default: Value, - _ keyPath: KeyPath + _ keyPath: KeyPath ) -> Value { if let providedValue { return providedValue } guard let test = Test.current else { return `default` } - guard let trait = test.traits.compactMap({ $0 as? TraitKind }).last else { - print("No traits of type \(TraitKind.self) found. Returning default.") - print("Traits: \(test.traits)") - return `default` - } - return trait[keyPath: keyPath] + let possibleTraits = test.traits.compactMap { $0 as? TraitKind } + let traitValues = possibleTraits.compactMap { $0[keyPath: keyPath] } + return traitValues.last ?? `default` } /// A type to record the last value returned by a closure returning an optional @@ -397,12 +399,16 @@ private struct Poller { isolation: isolated (any Actor)? = #isolation, expression: @escaping () async -> Bool ) async -> PollResult { - for _ in 0.. Self { - ConfirmPassesAlwaysConfigurationTrait( + ConfirmAlwaysPassesConfigurationTrait( maxPollingIterations: maxPollingIterations, pollingInterval: pollingInterval ) diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift index d6330372a..173ac8ecf 100644 --- a/Tests/TestingTests/PollingTests.swift +++ b/Tests/TestingTests/PollingTests.swift @@ -179,7 +179,7 @@ struct PollingTests { @Suite( "Configuration traits", - .confirmPassesAlwaysDefaults(maxPollingIterations: 100) + .confirmAlwaysPassesDefaults(maxPollingIterations: 100) ) struct WithConfigurationTraits { @Test("When no test or callsite configuration provided, uses the suite configuration") @@ -194,7 +194,7 @@ struct PollingTests { @Test( "When test configuration porvided, uses the test configuration", - .confirmPassesAlwaysDefaults(maxPollingIterations: 10) + .confirmAlwaysPassesDefaults(maxPollingIterations: 10) ) func testUsesTestConfigurationOverSuiteConfiguration() async { let incrementor = Incrementor() @@ -206,7 +206,7 @@ struct PollingTests { @Test( "When callsite configuration provided, uses that", - .confirmPassesAlwaysDefaults(maxPollingIterations: 10) + .confirmAlwaysPassesDefaults(maxPollingIterations: 10) ) func testUsesCallsiteConfiguration() async { let incrementor = Incrementor() @@ -218,10 +218,11 @@ struct PollingTests { } } - @Suite("Duration Tests", .disabled("time-sensitive")) struct DurationTests { + @Suite("Duration Tests", .disabled("time-sensitive")) + struct DurationTests { @Suite("confirmPassesEventually") struct PassesOnceBehavior { - let delta = Duration.seconds(6) + let delta = Duration.milliseconds(100) @Test("Simple passing expressions") func trivialHappyPath() async { let duration = await Test.Clock().measure { @@ -237,10 +238,11 @@ struct PollingTests { } #expect(issues.count == 1) } - #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + #expect(duration.isCloseTo(other: .seconds(2), within: delta)) } - @Test("When the value changes from false to true during execution") func changingFromFail() async { + @Test("When the value changes from false to true during execution") + func changingFromFail() async { let incrementor = Incrementor() let duration = await Test.Clock().measure { @@ -256,18 +258,36 @@ struct PollingTests { #expect(await incrementor.count == 2) #expect(duration.isCloseTo(other: .zero, within: delta)) } + + @Test("Doesn't wait after the last iteration") + func lastIteration() async { + let duration = await Test.Clock().measure { + let issues = await runTest { + await confirmPassesEventually( + maxPollingIterations: 10, + pollingInterval: .seconds(1) // Wait a long time to handle jitter. + ) { false } + } + #expect(issues.count == 1) + } + #expect( + duration.isCloseTo( + other: .seconds(9), + within: .milliseconds(500) + ) + ) + } } @Suite("confirmAlwaysPasses") struct PassesAlwaysBehavior { - // use a very generous delta for CI reasons. - let delta = Duration.seconds(6) + let delta = Duration.milliseconds(100) @Test("Simple passing expressions") func trivialHappyPath() async { let duration = await Test.Clock().measure { await confirmAlwaysPasses { true } } - #expect(duration.isCloseTo(other: .seconds(60), within: delta)) + #expect(duration.isCloseTo(other: .seconds(1), within: delta)) } @Test("Simple failing expressions") func trivialSadPath() async { @@ -279,6 +299,22 @@ struct PollingTests { } #expect(duration.isCloseTo(other: .zero, within: delta)) } + + @Test("Doesn't wait after the last iteration") + func lastIteration() async { + let duration = await Test.Clock().measure { + await confirmAlwaysPasses( + maxPollingIterations: 10, + pollingInterval: .seconds(1) // Wait a long time to handle jitter. + ) { true } + } + #expect( + duration.isCloseTo( + other: .seconds(9), + within: .milliseconds(500) + ) + ) + } } } } From bc01e1bc84f1e79d88a01e6580fbd6f727d059ec Mon Sep 17 00:00:00 2001 From: Rachel Brindle Date: Sat, 7 Jun 2025 22:11:04 -0700 Subject: [PATCH 6/6] Add requirePassesEventually and requireAlwaysPasses These two mirror their confirm counterparts, only throwing an error (instead of recording an issue) when they fail. --- Sources/Testing/Polling/Polling.swift | 164 ++++++++++++++++++++++++-- Tests/TestingTests/PollingTests.swift | 11 +- 2 files changed, 166 insertions(+), 9 deletions(-) diff --git a/Sources/Testing/Polling/Polling.swift b/Sources/Testing/Polling/Polling.swift index ffd3978f8..851e68ecd 100644 --- a/Sources/Testing/Polling/Polling.swift +++ b/Sources/Testing/Polling/Polling.swift @@ -14,6 +14,10 @@ internal let defaultPollingConfiguration = ( pollingInterval: Duration.milliseconds(1) ) +/// A type describing an error thrown when polling fails. +@_spi(Experimental) +public struct PollingFailedError: Error, Equatable {} + /// Confirm that some expression eventually returns true /// /// - Parameters: @@ -76,10 +80,73 @@ public func confirmPassesEventually( } } -/// A type describing an error thrown when polling fails to return a non-nil -/// value +/// Require that some expression eventually returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or +/// suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmPassesEventuallyConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmPassesEventuallyConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Throws: A `PollingFailedError` will be thrown if the expression never +/// returns true. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, waiting on some state to change that cannot be easily confirmed +/// through other forms of `confirmation`. @_spi(Experimental) -public struct PollingFailedError: Error {} +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func requirePassesEventually( + _ comment: Comment? = nil, + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async throws { + let poller = Poller( + pollingBehavior: .passesOnce, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmPassesEventuallyConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmPassesEventuallyConfigurationTrait.pollingInterval + ), + comment: comment, + sourceLocation: sourceLocation + ) + let passed = await poller.evaluate(raiseIssue: false, isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } + if !passed { + throw PollingFailedError() + } +} /// Confirm that some expression eventually returns a non-nil value /// @@ -108,7 +175,7 @@ public struct PollingFailedError: Error {} /// - Returns: The first non-nil value returned by `body`. /// /// - Throws: A `PollingFailedError` will be thrown if `body` never returns a -/// non-optional value +/// non-optional value. /// /// Use polling confirmations to check that an event while a test is running in /// complex scenarios where other forms of confirmation are insufficient. For @@ -215,6 +282,72 @@ public func confirmAlwaysPasses( } } +/// Require that some expression always returns true +/// +/// - Parameters: +/// - comment: An optional comment to apply to any issues generated by this +/// function. +/// - maxPollingIterations: The maximum amount of times to attempt polling. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then +/// polling will be attempted 1000 times before recording an issue. +/// `maxPollingIterations` must be greater than 0. +/// - pollingInterval: The minimum amount of time to wait between polling +/// attempts. +/// If nil, this uses whatever value is specified under the last +/// ``ConfirmAlwaysPassesConfigurationTrait`` added to the test or suite. +/// If no ``ConfirmAlwaysPassesConfigurationTrait`` has been added, then +/// polling will wait at least 1 millisecond between polling attempts. +/// `pollingInterval` must be greater than 0. +/// - isolation: The actor to which `body` is isolated, if any. +/// - sourceLocation: The source location to whych any recorded issues should +/// be attributed. +/// - body: The function to invoke. +/// +/// - Throws: A `PollingFailedError` will be thrown if the expression ever +/// returns false. +/// +/// Use polling confirmations to check that an event while a test is running in +/// complex scenarios where other forms of confirmation are insufficient. For +/// example, confirming that some state does not change. +@_spi(Experimental) +@available(macOS 13, iOS 17, watchOS 9, tvOS 17, visionOS 1, *) +public func requireAlwaysPasses( + _ comment: Comment? = nil, + maxPollingIterations: Int? = nil, + pollingInterval: Duration? = nil, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation = #_sourceLocation, + _ body: @escaping () async throws -> Bool +) async throws { + let poller = Poller( + pollingBehavior: .passesAlways, + pollingIterations: getValueFromPollingTrait( + providedValue: maxPollingIterations, + default: defaultPollingConfiguration.maxPollingIterations, + \ConfirmAlwaysPassesConfigurationTrait.maxPollingIterations + ), + pollingInterval: getValueFromPollingTrait( + providedValue: pollingInterval, + default: defaultPollingConfiguration.pollingInterval, + \ConfirmAlwaysPassesConfigurationTrait.pollingInterval + ), + comment: comment, + sourceLocation: sourceLocation + ) + let passed = await poller.evaluate(raiseIssue: false, isolation: isolation) { + do { + return try await body() + } catch { + return false + } + } + if !passed { + throw PollingFailedError() + } +} + /// A helper function to de-duplicate the logic of grabbing configuration from /// either the passed-in value (if given), the hardcoded default, and the /// appropriate configuration trait. @@ -368,23 +501,38 @@ private struct Poller { /// Evaluate polling, and process the result, raising an issue if necessary. /// /// - Parameters: + /// - raiseIssue: Whether or not to raise an issue. + /// This should only be false for `requirePassesEventually` or + /// `requireAlwaysPasses`. + /// - isolation: The isolation to use /// - body: The expression to poll + /// + /// - Returns: Whether or not polling passed. + /// /// - Side effects: If polling fails (see `PollingBehavior`), then this will /// record an issue. - func evaluate( + @discardableResult func evaluate( + raiseIssue: Bool = true, isolation: isolated (any Actor)?, _ body: @escaping () async -> Bool - ) async { + ) async -> Bool { precondition(pollingIterations > 0) precondition(pollingInterval > Duration.zero) let result = await poll( expression: body ) - result.issue( + if let issue = result.issue( comment: comment, sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation), pollingBehavior: pollingBehavior - )?.record() + ) { + if raiseIssue { + issue.record() + } + return false + } else { + return true + } } /// This function contains the logic for continuously polling an expression, diff --git a/Tests/TestingTests/PollingTests.swift b/Tests/TestingTests/PollingTests.swift index 173ac8ecf..4f97e15bd 100644 --- a/Tests/TestingTests/PollingTests.swift +++ b/Tests/TestingTests/PollingTests.swift @@ -16,8 +16,10 @@ struct PollingTests { struct PassesOnceBehavior { @Test("Simple passing expressions") func trivialHappyPath() async throws { await confirmPassesEventually { true } + try await requirePassesEventually { true } let value = try await confirmPassesEventually { 1 } + #expect(value == 1) } @@ -25,6 +27,9 @@ struct PollingTests { let issues = await runTest { await confirmPassesEventually { false } _ = try await confirmPassesEventually { Optional.none } + await #expect(throws: PollingFailedError()) { + try await requirePassesEventually { false } + } } #expect(issues.count == 3) } @@ -122,13 +127,17 @@ struct PollingTests { @Suite("confirmAlwaysPasses") struct PassesAlwaysBehavior { - @Test("Simple passing expressions") func trivialHappyPath() async { + @Test("Simple passing expressions") func trivialHappyPath() async throws { await confirmAlwaysPasses { true } + try await requireAlwaysPasses { true } } @Test("Simple failing expressions") func trivialSadPath() async { let issues = await runTest { await confirmAlwaysPasses { false } + await #expect(throws: PollingFailedError()) { + try await requireAlwaysPasses { false } + } } #expect(issues.count == 1) }