From b38cfb392790f331b439f103a5d83fed7b933f52 Mon Sep 17 00:00:00 2001 From: Suzy Ratcliff Date: Thu, 10 Apr 2025 14:05:24 -0700 Subject: [PATCH 1/4] Introduce a severity level when recording issues --- Sources/Testing/Issues/Issue+Recording.swift | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index aaf721c6a..7ce3ff0bb 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -107,6 +107,34 @@ extension Issue { let issue = Issue(kind: .errorCaught(error), comments: Array(comment), sourceContext: sourceContext) return issue.record() } + + /// Record a new issue when a running test unexpectedly catches an error. + /// + /// - Parameters: + /// - error: The error that caused the issue. + /// - comment: A comment describing the expectation. + /// - sourceLocation: The source location to which the issue should be + /// attributed. + /// - severity: The severity of the issue. + /// + /// - Returns: The issue that was recorded. + /// + /// This function can be used if an unexpected error is caught while running a + /// test and it should be treated as a test failure. If an error is thrown + /// from a test function, it is automatically recorded as an issue and this + /// function does not need to be used. + @_spi(Experimental) + @discardableResult public static func record( + _ error: any Error, + _ comment: Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + severity: Severity = .error, + ) -> Self { + let backtrace = Backtrace(forFirstThrowOf: error) ?? Backtrace.current() + let sourceContext = SourceContext(backtrace: backtrace, sourceLocation: sourceLocation) + let issue = Issue(kind: .errorCaught(error), severity: severity, comments: Array(comment), sourceContext: sourceContext) + return issue.record() + } /// Catch any error thrown from a closure and record it as an issue instead of /// allowing it to propagate to the caller. From 3c219ba0cccada13c8a4f4b41629bc8694193996 Mon Sep 17 00:00:00 2001 From: Suzy Ratcliff Date: Thu, 10 Apr 2025 14:48:15 -0700 Subject: [PATCH 2/4] Fixes: rdar://149018685. Refactor code for sharing and write tests --- Sources/Testing/Issues/Issue+Recording.swift | 35 ++++++++++++---- Tests/TestingTests/IssueTests.swift | 42 ++++++++++++++++++++ 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index 7ce3ff0bb..59657d449 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -74,9 +74,31 @@ extension Issue { _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation ) -> Self { - let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) - let issue = Issue(kind: .unconditional, comments: Array(comment), sourceContext: sourceContext) - return issue.record() + return Self.record(comment, sourceLocation: sourceLocation, severity: .error) + } + + /// Record an issue when a running test fails unexpectedly. + /// + /// - Parameters: + /// - comment: A comment describing the expectation. + /// - sourceLocation: The source location to which the issue should be + /// attributed. + /// - severity: The severity of the issue. + /// + /// - Returns: The issue that was recorded. + /// + /// Use this function if, while running a test, an issue occurs that cannot be + /// represented as an expectation (using the ``expect(_:_:sourceLocation:)`` + /// or ``require(_:_:sourceLocation:)-5l63q`` macros.) + @_spi(Experimental) + @discardableResult public static func record( + _ comment: Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation, + severity: Severity + ) -> Self { + let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) + let issue = Issue(kind: .unconditional, severity: severity, comments: Array(comment), sourceContext: sourceContext) + return issue.record() } } @@ -102,10 +124,7 @@ extension Issue { _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation ) -> Self { - let backtrace = Backtrace(forFirstThrowOf: error) ?? Backtrace.current() - let sourceContext = SourceContext(backtrace: backtrace, sourceLocation: sourceLocation) - let issue = Issue(kind: .errorCaught(error), comments: Array(comment), sourceContext: sourceContext) - return issue.record() + return Self.record(error, comment, sourceLocation: sourceLocation, severity: .error) } /// Record a new issue when a running test unexpectedly catches an error. @@ -128,7 +147,7 @@ extension Issue { _ error: any Error, _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation, - severity: Severity = .error, + severity: Severity ) -> Self { let backtrace = Backtrace(forFirstThrowOf: error) ?? Backtrace.current() let sourceContext = SourceContext(backtrace: backtrace, sourceLocation: sourceLocation) diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index d22bf9fba..da74d3e0f 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -1010,6 +1010,7 @@ final class IssueTests: XCTestCase { return } XCTAssertFalse(issue.isKnown) + XCTAssertTrue(issue.severity == .error) guard case .unconditional = issue.kind else { XCTFail("Unexpected issue kind \(issue.kind)") return @@ -1021,6 +1022,25 @@ final class IssueTests: XCTestCase { Issue.record("Custom message") }.run(configuration: configuration) } + + func testWarning() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + XCTAssertFalse(issue.isKnown) + XCTAssertTrue(issue.severity == .warning) + guard case .unconditional = issue.kind else { + XCTFail("Unexpected issue kind \(issue.kind)") + return + } + } + + await Test { + Issue.record("Custom message", severity: .warning) + }.run(configuration: configuration) + } #if !SWT_NO_UNSTRUCTURED_TASKS func testFailWithoutCurrentTest() async throws { @@ -1048,6 +1068,7 @@ final class IssueTests: XCTestCase { return } XCTAssertFalse(issue.isKnown) + XCTAssertTrue(issue.severity == .error) guard case let .errorCaught(error) = issue.kind else { XCTFail("Unexpected issue kind \(issue.kind)") return @@ -1060,6 +1081,27 @@ final class IssueTests: XCTestCase { Issue.record(MyError(), "Custom message") }.run(configuration: configuration) } + + func testWarningBecauseOfError() async throws { + var configuration = Configuration() + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { + return + } + XCTAssertFalse(issue.isKnown) + XCTAssertTrue(issue.severity == .warning) + guard case let .errorCaught(error) = issue.kind else { + XCTFail("Unexpected issue kind \(issue.kind)") + return + } + XCTAssertTrue(error is MyError) + } + + await Test { + Issue.record(MyError(), severity: .warning) + Issue.record(MyError(), "Custom message", severity: .warning) + }.run(configuration: configuration) + } func testErrorPropertyValidForThrownErrors() async throws { var configuration = Configuration() From 362265c1ce4c420c8a3836403847c7b039f56d5d Mon Sep 17 00:00:00 2001 From: Suzy Ratcliff Date: Thu, 10 Apr 2025 14:56:07 -0700 Subject: [PATCH 3/4] Match style guide --- Sources/Testing/Issues/Issue+Recording.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index 59657d449..96b6f577a 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -74,7 +74,7 @@ extension Issue { _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation ) -> Self { - return Self.record(comment, sourceLocation: sourceLocation, severity: .error) + record(comment, sourceLocation: sourceLocation, severity: .error) } /// Record an issue when a running test fails unexpectedly. @@ -124,7 +124,7 @@ extension Issue { _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation ) -> Self { - return Self.record(error, comment, sourceLocation: sourceLocation, severity: .error) + record(error, comment, sourceLocation: sourceLocation, severity: .error) } /// Record a new issue when a running test unexpectedly catches an error. From 925384856edb699746d82a9329347a02af6c2a92 Mon Sep 17 00:00:00 2001 From: Suzy Ratcliff Date: Thu, 10 Apr 2025 15:03:23 -0700 Subject: [PATCH 4/4] Fixes: rdar://149018685. Update to match style guide --- Sources/Testing/Issues/Issue+Recording.swift | 22 ++++++++++---------- Tests/TestingTests/IssueTests.swift | 8 +++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index 96b6f577a..bd3e9a3bb 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -74,16 +74,16 @@ extension Issue { _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation ) -> Self { - record(comment, sourceLocation: sourceLocation, severity: .error) + record(comment, severity: .error, sourceLocation: sourceLocation) } /// Record an issue when a running test fails unexpectedly. /// /// - Parameters: /// - comment: A comment describing the expectation. + /// - severity: The severity of the issue. /// - sourceLocation: The source location to which the issue should be /// attributed. - /// - severity: The severity of the issue. /// /// - Returns: The issue that was recorded. /// @@ -93,12 +93,12 @@ extension Issue { @_spi(Experimental) @discardableResult public static func record( _ comment: Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - severity: Severity + severity: Severity, + sourceLocation: SourceLocation = #_sourceLocation ) -> Self { - let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) - let issue = Issue(kind: .unconditional, severity: severity, comments: Array(comment), sourceContext: sourceContext) - return issue.record() + let sourceContext = SourceContext(backtrace: .current(), sourceLocation: sourceLocation) + let issue = Issue(kind: .unconditional, severity: severity, comments: Array(comment), sourceContext: sourceContext) + return issue.record() } } @@ -124,7 +124,7 @@ extension Issue { _ comment: Comment? = nil, sourceLocation: SourceLocation = #_sourceLocation ) -> Self { - record(error, comment, sourceLocation: sourceLocation, severity: .error) + record(error, comment, severity: .error, sourceLocation: sourceLocation) } /// Record a new issue when a running test unexpectedly catches an error. @@ -132,9 +132,9 @@ extension Issue { /// - Parameters: /// - error: The error that caused the issue. /// - comment: A comment describing the expectation. + /// - severity: The severity of the issue. /// - sourceLocation: The source location to which the issue should be /// attributed. - /// - severity: The severity of the issue. /// /// - Returns: The issue that was recorded. /// @@ -146,8 +146,8 @@ extension Issue { @discardableResult public static func record( _ error: any Error, _ comment: Comment? = nil, - sourceLocation: SourceLocation = #_sourceLocation, - severity: Severity + severity: Severity, + sourceLocation: SourceLocation = #_sourceLocation ) -> Self { let backtrace = Backtrace(forFirstThrowOf: error) ?? Backtrace.current() let sourceContext = SourceContext(backtrace: backtrace, sourceLocation: sourceLocation) diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index da74d3e0f..cb7ce28f3 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -1010,7 +1010,7 @@ final class IssueTests: XCTestCase { return } XCTAssertFalse(issue.isKnown) - XCTAssertTrue(issue.severity == .error) + XCTAssertEqual(issue.severity, .error) guard case .unconditional = issue.kind else { XCTFail("Unexpected issue kind \(issue.kind)") return @@ -1030,7 +1030,7 @@ final class IssueTests: XCTestCase { return } XCTAssertFalse(issue.isKnown) - XCTAssertTrue(issue.severity == .warning) + XCTAssertEqual(issue.severity, .warning) guard case .unconditional = issue.kind else { XCTFail("Unexpected issue kind \(issue.kind)") return @@ -1068,7 +1068,7 @@ final class IssueTests: XCTestCase { return } XCTAssertFalse(issue.isKnown) - XCTAssertTrue(issue.severity == .error) + XCTAssertEqual(issue.severity, .error) guard case let .errorCaught(error) = issue.kind else { XCTFail("Unexpected issue kind \(issue.kind)") return @@ -1089,7 +1089,7 @@ final class IssueTests: XCTestCase { return } XCTAssertFalse(issue.isKnown) - XCTAssertTrue(issue.severity == .warning) + XCTAssertEqual(issue.severity, .warning) guard case let .errorCaught(error) = issue.kind else { XCTFail("Unexpected issue kind \(issue.kind)") return