From 8da6950c3d9c1756eee19395b1fdf1ec37a2b85e Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 13 Nov 2024 12:45:41 -0600 Subject: [PATCH 1/4] Expand the 'Known Issues' documentation article Resolves rdar://137961874 --- Sources/Testing/Issues/KnownIssue.swift | 24 +++- Sources/Testing/Testing.docc/known-issues.md | 130 ++++++++++++++++++- 2 files changed, 146 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/Issues/KnownIssue.swift b/Sources/Testing/Issues/KnownIssue.swift index 4d7c16739..f59e9d422 100644 --- a/Sources/Testing/Issues/KnownIssue.swift +++ b/Sources/Testing/Issues/KnownIssue.swift @@ -96,7 +96,7 @@ public typealias KnownIssueMatcher = @Sendable (_ issue: Issue) -> Bool /// be attributed. /// - body: The function to invoke. /// -/// Use this function when a test is known to raise one or more issues that +/// Use this function when a test is known to record one or more issues that /// should not cause the test to fail. For example: /// /// ```swift @@ -112,6 +112,10 @@ public typealias KnownIssueMatcher = @Sendable (_ issue: Issue) -> Bool /// while others should continue to cause test failures, use /// ``withKnownIssue(_:isIntermittent:sourceLocation:_:when:matching:)`` /// instead. +/// +/// ## See Also +/// +/// - public func withKnownIssue( _ comment: Comment? = nil, isIntermittent: Bool = false, @@ -143,7 +147,7 @@ public func withKnownIssue( /// - Throws: Whatever is thrown by `body`, unless it is matched by /// `issueMatcher`. /// -/// Use this function when a test is known to raise one or more issues that +/// Use this function when a test is known to record one or more issues that /// should not cause the test to fail, or if a precondition affects whether /// issues are known to occur. For example: /// @@ -165,6 +169,10 @@ public func withKnownIssue( /// instead. /// /// - Note: `issueMatcher` may be invoked more than once for the same issue. +/// +/// ## See Also +/// +/// - public func withKnownIssue( _ comment: Comment? = nil, isIntermittent: Bool = false, @@ -205,7 +213,7 @@ public func withKnownIssue( /// be attributed. /// - body: The function to invoke. /// -/// Use this function when a test is known to raise one or more issues that +/// Use this function when a test is known to record one or more issues that /// should not cause the test to fail. For example: /// /// ```swift @@ -221,6 +229,10 @@ public func withKnownIssue( /// while others should continue to cause test failures, use /// ``withKnownIssue(_:isIntermittent:isolation:sourceLocation:_:when:matching:)`` /// instead. +/// +/// ## See Also +/// +/// - public func withKnownIssue( _ comment: Comment? = nil, isIntermittent: Bool = false, @@ -254,7 +266,7 @@ public func withKnownIssue( /// - Throws: Whatever is thrown by `body`, unless it is matched by /// `issueMatcher`. /// -/// Use this function when a test is known to raise one or more issues that +/// Use this function when a test is known to record one or more issues that /// should not cause the test to fail, or if a precondition affects whether /// issues are known to occur. For example: /// @@ -276,6 +288,10 @@ public func withKnownIssue( /// instead. /// /// - Note: `issueMatcher` may be invoked more than once for the same issue. +/// +/// ## See Also +/// +/// - public func withKnownIssue( _ comment: Comment? = nil, isIntermittent: Bool = false, diff --git a/Sources/Testing/Testing.docc/known-issues.md b/Sources/Testing/Testing.docc/known-issues.md index 31906a5df..19b3ae4f0 100644 --- a/Sources/Testing/Testing.docc/known-issues.md +++ b/Sources/Testing/Testing.docc/known-issues.md @@ -10,13 +10,135 @@ See https://swift.org/LICENSE.txt for license information See https://swift.org/CONTRIBUTORS.txt for Swift project authors --> -Highlight known issues when running tests. +Mark issues as known when running tests. ## Overview -The testing library provides a function, `withKnownIssue()`, that you -use to mark issues as known. Use this function to inform the testing library -at runtime not to mark the test as failing when issues occur. +The testing library provides several functions named `withKnownIssue()` that +you can use to mark issues as known. Use them to inform the testing library not +to mark a test as failing if only known issues are recorded. + +### Mark an expectation failure as known + +Consider a test function with a single expectation: + +```swift +@Test func grillHeating() throws { + var foodTruck = FoodTruck() + try foodTruck.startGrill() + #expect(foodTruck.grill.isHeating) // ❌ Expectation failed +} +``` + +If the value of the `isHeating` property is `false`, `#expect` will record an +issue. If you cannot fix the underlying problem, you can surround the failing +code in a closure passed to `withKnownIssue()`: + +```swift +@Test func grillHeating() throws { + var foodTruck = FoodTruck() + try foodTruck.startGrill() + withKnownIssue("Propane tank is empty") { + #expect(foodTruck.grill.isHeating) // Known issue + } +} +``` + +The issue recorded by `#expect` will then be considered "known" and the test +will not be marked as a failure. You may include an optional comment to explain +the problem or provide context. + +### Mark a thrown error as known + +If an `Error` is caught by the closure passed to `withKnownIssue()`, the issue +representing that caught error will be marked as known. Continuing the previous +example, suppose the problem is that the `startGrill()` function is throwing an +error. You can apply `withKnownIssue()` to this situation as well: + +```swift +@Test func grillHeating() { + var foodTruck = FoodTruck() + withKnownIssue { + try foodTruck.startGrill() // Known issue + #expect(foodTruck.grill.isHeating) + } +} +``` + +Because all errors thrown from the closure are caught and interpreted as known +issues, the `withKnownIssue()` function is not throwing. Consequently, any +subsequent code which depends on the throwing call having succeeded (such as the +`#expect` after `startGrill()`) must be included in the closure to avoid +additional issues. + +### Match a specific issue + +By default, `withKnownIssue()` considers all issues recorded while invoking the +body closure known. If multiple issues may be recorded, you can pass a trailing +closure labeled `matching:` which will be called once for each recorded issue +to determine whether it should be treated as known: + +```swift +@Test func batteryLevel() throws { + var foodTruck = FoodTruck() + try withKnownIssue { + let batteryLevel = try #require(foodTruck.batteryLevel) // Known + #expect(batteryLevel >= 0.8) // Not considered known + } matching: { issue in + guard case .expectationFailed(let expectation) = issue.kind else { + return false + } + return expectation.isRequired + } +} +``` + +### Resolve a known issue + +If there are no issues recorded while calling `function`, `withKnownIssue()` +will record a distinct issue about the lack of any issues having been recorded. +This notifies you that the underlying problem may have been resolved so that you +can investigate and consider removing `withKnownIssue()` if it's no longer +necessary. + +### Handle a nondeterministic failure + +If `withKnownIssue()` sometimes succeeds but other times records an issue +indicating there were no known issues, this may indicate a nondeterministic +failure or a "flaky" test. + +If the underlying problem only occurs in certain circumstances, consider +including a precondition. For example, if the grill only fails to heat when +there's no propane, you can pass a trailing closure labeled `when:` which +determines whether issues recorded in the body closure should be considered +known: + +```swift +@Test func grillHeating() throws { + var foodTruck = FoodTruck() + try foodTruck.startGrill() + withKnownIssue { + // Only considered known when hasPropane == false + #expect(foodTruck.grill.isHeating) + } when: { + !hasPropane + } +} +``` + +If the underlying problem is truly nondeterministic, you may acknowledge this +and instruct `withKnownIssue()` to not record an issue indicating there were no +known issues by passing `isIntermittent: true`: + +```swift +@Test func grillHeating() throws { + var foodTruck = FoodTruck() + try foodTruck.startGrill() + withKnownIssue(isIntermittent: true) { + #expect(foodTruck.grill.isHeating) + } +} +``` ## Topics From ac5b70dc735863d644783e01487957ccb43307a1 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 13 Nov 2024 13:12:26 -0600 Subject: [PATCH 2/4] Improve grammar/phrasing in overview --- Sources/Testing/Testing.docc/known-issues.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Testing.docc/known-issues.md b/Sources/Testing/Testing.docc/known-issues.md index 19b3ae4f0..d7df368d6 100644 --- a/Sources/Testing/Testing.docc/known-issues.md +++ b/Sources/Testing/Testing.docc/known-issues.md @@ -15,8 +15,8 @@ Mark issues as known when running tests. ## Overview The testing library provides several functions named `withKnownIssue()` that -you can use to mark issues as known. Use them to inform the testing library not -to mark a test as failing if only known issues are recorded. +you can use to mark issues as known. Use them to inform the testing library that +a test should not be marked as failing if only known issues are recorded. ### Mark an expectation failure as known From 90435828632bd6bbdcbcc2319f9ce0017ddd642a Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 19 Nov 2024 11:20:41 -0600 Subject: [PATCH 3/4] Incorporate review feedback --- Sources/Testing/Testing.docc/known-issues.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Sources/Testing/Testing.docc/known-issues.md b/Sources/Testing/Testing.docc/known-issues.md index d7df368d6..736eda7ea 100644 --- a/Sources/Testing/Testing.docc/known-issues.md +++ b/Sources/Testing/Testing.docc/known-issues.md @@ -71,6 +71,10 @@ subsequent code which depends on the throwing call having succeeded (such as the `#expect` after `startGrill()`) must be included in the closure to avoid additional issues. +- Note: `withKnownIssue()` is recommended instead of `#expect(throws:)` for any + error which is considered a known issue so that the test status and results + will reflect the situation more accurately. + ### Match a specific issue By default, `withKnownIssue()` considers all issues recorded while invoking the @@ -107,6 +111,12 @@ If `withKnownIssue()` sometimes succeeds but other times records an issue indicating there were no known issues, this may indicate a nondeterministic failure or a "flaky" test. +The first step in resolving a nondeterministic test failure is to analyze the +code being tested and determine the source of the unpredictable behavior. If +you discover a bug, such as a race condition, the ideal resolution is to fix +that underlying problem so that the code always behaves consistently, even if +it continues to exhibit the known issue. + If the underlying problem only occurs in certain circumstances, consider including a precondition. For example, if the grill only fails to heat when there's no propane, you can pass a trailing closure labeled `when:` which @@ -126,9 +136,10 @@ known: } ``` -If the underlying problem is truly nondeterministic, you may acknowledge this -and instruct `withKnownIssue()` to not record an issue indicating there were no -known issues by passing `isIntermittent: true`: +If the underlying problem is unpredictable and fails at random, you can pass +`isIntermittent: true` to let the testing library know that it will not always +occur. Then, the testing library will _not_ record the issue indicating that no +known issues were recorded: ```swift @Test func grillHeating() throws { From 3d463e11e304f753b3af325dea54cbdbd6fda4c3 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 19 Nov 2024 11:32:46 -0600 Subject: [PATCH 4/4] More edits --- Sources/Testing/Testing.docc/known-issues.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/Testing.docc/known-issues.md b/Sources/Testing/Testing.docc/known-issues.md index 736eda7ea..e400e3032 100644 --- a/Sources/Testing/Testing.docc/known-issues.md +++ b/Sources/Testing/Testing.docc/known-issues.md @@ -113,8 +113,8 @@ failure or a "flaky" test. The first step in resolving a nondeterministic test failure is to analyze the code being tested and determine the source of the unpredictable behavior. If -you discover a bug, such as a race condition, the ideal resolution is to fix -that underlying problem so that the code always behaves consistently, even if +you discover a bug such as a race condition, the ideal resolution is to fix +the underlying problem so that the code always behaves consistently even if it continues to exhibit the known issue. If the underlying problem only occurs in certain circumstances, consider @@ -138,8 +138,8 @@ known: If the underlying problem is unpredictable and fails at random, you can pass `isIntermittent: true` to let the testing library know that it will not always -occur. Then, the testing library will _not_ record the issue indicating that no -known issues were recorded: +occur. Then, the testing library will not record an issue when zero known issues +are recorded: ```swift @Test func grillHeating() throws {