Updates tests for execution on non Apple hosts. (#8407)

- Update StringChecker to handle Windows line endings.
- Use .exe suffix for executables on Windows
- Use where.exe on Windows for tool location detection.
- Enable unconditionally skipped tests, marking with known issue where
appropriate.
- Fix unconditionally skipped tests, skipping was added many years ago
and never removed.
- Add matrix style testing for the build system (native/swiftbuild) on
"swift init/build/test" tests.

Author: kcieplak

IntegrationTests/Sources/IntegrationTestSupport/Helpers.swift

@@ -37,6 +37,8 @@ public let toolchainPath: AbsolutePath = {
     let swiftcPath = try! AbsolutePath(
         validating: sh("xcrun", "--find", "swift").stdout.spm_chomp()
     )
+    #elseif os(Windows)
+    let swiftcPath = try! AbsolutePath(validating: sh("where.exe", "swift.exe").stdout.spm_chomp())
     #else
     let swiftcPath = try! AbsolutePath(validating: sh("which", "swift").stdout.spm_chomp())
     #endif

IntegrationTests/Sources/IntegrationTestSupport/Process.swift

@@ -28,4 +28,16 @@ extension ProcessInfo {
     #else
     public static let hostOperatingSystem = OperatingSystem.unknown
     #endif
+
+    #if os(Windows)
+    public static let EOL = "\r\n"
+    #else
+    public static let EOL = "\n"
+    #endif
+
+    #if os(Windows)
+    public static let exeSuffix = ".exe"
+    #else
+    public static let exeSuffix = ""
+    #endif
 }

IntegrationTests/Sources/IntegrationTestSupport/StringChecker.swift

@@ -18,7 +18,7 @@ public class StringChecker {
 
     public init(string: String) {
         self.string = string
-        self.lines = string.split(separator: "\n")
+        self.lines = string.split(separator: ProcessInfo.EOL)
         self.currentLineIndex = 0
     }
 

IntegrationTests/Tests/IntegrationTests/BasicTests.swift

@@ -8,6 +8,7 @@
  See http://swift.org/CONTRIBUTORS.txt for Swift project authors
  */
 
+import Foundation
 import IntegrationTestSupport
 import Testing
 import TSCBasic
@@ -25,13 +26,15 @@ private struct BasicTests {
         .skipSwiftCISelfHosted(
             "These packages don't use the latest runtime library, which doesn't work with self-hosted builds."
         ),
-        .requireUnrestrictedNetworkAccess("Test requires access to https://github.com")
+        .requireUnrestrictedNetworkAccess("Test requires access to https://github.com"),
+        .skipHostOS(.windows, "Issue #8409 - random.swift:34:8: error: unsupported platform")
     )
     func testExamplePackageDealer() throws {
         try withTemporaryDirectory { tempDir in
             let packagePath = tempDir.appending(component: "dealer")
-            try sh("git", "clone", "https://github.com/apple/example-package-dealer", packagePath)
+            try sh("git\(ProcessInfo.exeSuffix)", "clone", "https://github.com/apple/example-package-dealer", packagePath)
             let build1Output = try sh(swiftBuild, "--package-path", packagePath).stdout
+
             // Check the build log.
             #expect(build1Output.contains("Build complete"))
 
@@ -43,7 +46,7 @@ private struct BasicTests {
 
             // Verify that the 'git status' is clean after a build.
             try localFileSystem.changeCurrentWorkingDirectory(to: packagePath)
-            let gitOutput = try sh("git", "status").stdout
+            let gitOutput = try sh("git\(ProcessInfo.exeSuffix)", "status").stdout
             #expect(gitOutput.contains("nothing to commit, working tree clean"))
 
             // Verify that another 'swift build' does nothing.
@@ -84,9 +87,9 @@ private struct BasicTests {
             #expect(try #/swiftc.* -module-name tool/#.firstMatch(in: buildOutput) != nil)
 
             // Verify that the tool exists and works.
-            let toolOutput = try sh(packagePath.appending(components: ".build", "debug", "tool")).stdout
-            #expect(toolOutput.contains("HI"))
-            #expect(toolOutput.contains("\n"))
+            let toolOutput = try sh(packagePath.appending(components: ".build", "debug", "tool"))
+                .stdout
+            #expect(toolOutput == "HI\(ProcessInfo.EOL)")
         }
     }
 
@@ -108,7 +111,7 @@ private struct BasicTests {
 
             // Check the file runs.
             let helloOutput = try sh(helloBinaryPath).stdout
-            #expect(helloOutput == "hello\n")
+            #expect(helloOutput == "hello\(ProcessInfo.EOL)")
         }
     }
 
@@ -140,24 +143,26 @@ private struct BasicTests {
         }
     }
 
-    @Test(.skip("FIXME: swift-test invocations are timing out in Xcode and self-hosted CI"))
+    @Test
     func testSwiftPackageInitExecTests() throws {
         try withTemporaryDirectory { tempDir in
             // Create a new package with an executable target.
             let packagePath = tempDir.appending(component: "Project")
             try localFileSystem.createDirectory(packagePath)
-            try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "executable")
-            let testOutput = try sh(swiftTest, "--package-path", packagePath).stdout
-
-            // Check the test log.
-            let checker = StringChecker(string: testOutput)
-            #expect(checker.check(.regex("Compiling .*ProjectTests.*")))
-            #expect(checker.check("Test Suite 'All tests' passed"))
-            #expect(checker.checkNext("Executed 1 test"))
-
-            // Check there were no compile errors or warnings.
-            #expect(testOutput.contains("error") == false)
-            #expect(testOutput.contains("warning") == false)
+            withKnownIssue("error: no tests found; create a target in the 'Tests' directory") {
+                try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "executable")
+                let testOutput = try sh(swiftTest, "--package-path", packagePath).stdout
+
+                // Check the test log.
+                let checker = StringChecker(string: testOutput)
+                #expect(checker.check(.regex("Compiling .*ProjectTests.*")))
+                #expect(checker.check("Test Suite 'All tests' passed"))
+                #expect(checker.checkNext("Executed 1 test"))
+
+                // Check there were no compile errors or warnings.
+                #expect(testOutput.contains("error") == false)
+                #expect(testOutput.contains("warning") == false)
+            }
         }
     }
 
@@ -180,7 +185,7 @@ private struct BasicTests {
         }
     }
 
-    @Test(.skip("FIXME: swift-test invocations are timing out in Xcode and self-hosted CI"))
+    @Test
     func testSwiftPackageLibsTests() throws {
         try withTemporaryDirectory { tempDir in
             // Create a new package with an executable target.
@@ -191,9 +196,9 @@ private struct BasicTests {
 
             // Check the test log.
             let checker = StringChecker(string: testOutput)
-            #expect(checker.check(.regex("Compiling .*ProjectTests.*")))
-            #expect(checker.check("Test Suite 'All tests' passed"))
-            #expect(checker.checkNext("Executed 1 test"))
+            #expect(checker.check(.contains("Test Suite 'All tests' started")))
+            #expect(checker.check(.contains("Test example() passed after")))
+            #expect(checker.checkNext(.contains("Test run with 1 test passed after")))
 
             // Check there were no compile errors or warnings.
             #expect(testOutput.contains("error") == false)
@@ -235,17 +240,19 @@ private struct BasicTests {
 
             // Check the build.
             let buildOutput = try sh(swiftBuild, "--package-path", packagePath, "-v").stdout
-            #expect(try 
-                #/swiftc.* -module-name special_tool .* '@.*/more spaces/special tool/.build/[^/]+/debug/special_tool.build/sources'/#
-                    .firstMatch(in: buildOutput) != nil
-            )
+            let expression = ProcessInfo
+                .hostOperatingSystem != .windows ?
+                #/swiftc.* -module-name special_tool .* '@.*/more spaces/special tool/.build/[^/]+/debug/special_tool.build/sources'/# :
+                #/swiftc.* -module-name special_tool .* "@.*\\more spaces\\special tool\\.build\\[^\\]+\\debug\\special_tool.build\\sources"/#
+            #expect(try expression.firstMatch(in: buildOutput) != nil)
             #expect(buildOutput.contains("Build complete"))
 
             // Verify that the tool exists and works.
             let toolOutput = try sh(
                 packagePath.appending(components: ".build", "debug", "special tool")
             ).stdout
-            #expect(toolOutput == "HI\n")
+
+            #expect(toolOutput == "HI\(ProcessInfo.EOL)")
         }
     }
 
@@ -284,11 +291,10 @@ private struct BasicTests {
             #expect(checker.check(.regex("Linking .*secho")))
             #expect(checker.check(.contains("Build of product 'secho' complete")))
 
-            #expect(runOutput == "1 \"two\"\n")
+            #expect(runOutput == "1 \"two\"\(ProcessInfo.EOL)")
         }
     }
 
-    @Test(.skip("FIXME: swift-test invocations are timing out in Xcode and self-hosted CI"))
     func testSwiftTest() throws {
         try withTemporaryDirectory { tempDir in
             let packagePath = tempDir.appending(component: "swiftTest")
@@ -325,7 +331,7 @@ private struct BasicTests {
         }
     }
 
-    @Test(.skip("FIXME: swift-test invocations are timing out in Xcode and self-hosted CI"))
+    @Test
     func testSwiftTestWithResources() throws {
         try withTemporaryDirectory { tempDir in
             let packagePath = tempDir.appending(component: "swiftTestResources")
@@ -406,7 +412,7 @@ private struct BasicTests {
 
             let testOutput = try sh(
                 swiftTest, "--package-path", packagePath, "--filter", "MyTests.*"
-            ).stderr
+            ).stdout
 
             // Check the test log.
             let checker = StringChecker(string: testOutput)

IntegrationTests/Tests/IntegrationTests/SwiftPMTests.swift

@@ -14,6 +14,12 @@ import Testing
 import TSCBasic
 import TSCTestSupport
 
+// TODO: This should be replaced with BuildSystem.BuildSystemProvider if 'IntegrationTests' are moved up under the swift-package-manager tests.
+public enum BuildSystemProvider: String, Codable, CaseIterable {
+    case native
+    case swiftbuild
+}
+
 @Suite
 private struct SwiftPMTests {
     @Test(.requireHostOS(.macOS))
@@ -69,46 +75,64 @@ private struct SwiftPMTests {
     }
 
     @Test(
-        .requireThreadSafeWorkingDirectory
+        .requireThreadSafeWorkingDirectory,
+        .bug(
+            "https://github.com/swiftlang/swift-package-manager/issues/8416",
+            "swift run using --build-system swiftbuild fails to run executable"
+        ),
+        arguments: BuildSystemProvider.allCases
     )
-    func packageInitExecutable() throws {
+    func packageInitExecutable(_ buildSystemProvider: BuildSystemProvider) throws {
         // Executable
         do {
             try withTemporaryDirectory { tmpDir in
                 let packagePath = tmpDir.appending(component: "foo")
                 try localFileSystem.createDirectory(packagePath)
                 try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "executable")
-                try sh(swiftBuild, "--package-path", packagePath, "--build-system", "swiftbuild")
-                // SWBINTTODO: Path issues related to swift test of the output from a swiftbuild buildsystem
-                // let (stdout, stderr) = try sh(
-                //     swiftRun, "--package-path", packagePath, "--build-system", "swiftbuild"
-                // )
-                // #expect(!stderr.contains("error:"))
-                // #expect(stdout.contains("Hello, world!"))
+                try sh(swiftBuild, "--package-path", packagePath, "--build-system", buildSystemProvider.rawValue)
+
+                try withKnownIssue("Error while loading shared libraries: libswiftCore.so: cannot open shared object file: No such file or directory") {
+                    // The 'native' build system uses 'swiftc' as the linker driver, which adds an RUNPATH to the swift runtime libraries in the SDK.
+                    // 'swiftbuild' directly calls clang, which does not add the extra RUNPATH, so runtime libraries cannot be found.
+                    let (stdout, stderr) = try sh(
+                        swiftRun, "--package-path", packagePath, "--build-system", buildSystemProvider.rawValue
+                    )
+                    #expect(!stderr.contains("error:"))
+                    #expect(stdout.contains("Hello, world!"))
+                } when: {
+                    buildSystemProvider == .swiftbuild && ProcessInfo.hostOperatingSystem == .linux
+                }
             }
         }
     }
 
     @Test(
-        .skipHostOS(
-            .windows,
-            "Windows fails to link this library package due to a 'lld-link: error: subsystem must be defined' error. See https://github.com/swiftlang/swift-build/issues/310"
-        ),
-        .requireThreadSafeWorkingDirectory
+        .requireThreadSafeWorkingDirectory,
+        .bug(id: 0, "SWBINTTODO: Linux: /lib/x86_64-linux-gnu/Scrt1.o:function _start: error:"),
+        .bug("https://github.com/swiftlang/swift-package-manager/issues/8380", "lld-link: error: subsystem must be defined"),
+        .bug(id:0, "SWBINTTODO: MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found"),
+        arguments: BuildSystemProvider.allCases
     )
-    func packageInitLibrary() throws {
+    func packageInitLibrary(_ buildSystemProvider: BuildSystemProvider) throws {
         do {
             try withTemporaryDirectory { tmpDir in
                 let packagePath = tmpDir.appending(component: "foo")
                 try localFileSystem.createDirectory(packagePath)
                 try sh(swiftPackage, "--package-path", packagePath, "init", "--type", "library")
-                try sh(swiftBuild, "--package-path", packagePath, "--build-system", "swiftbuild")
-                // SWBINTTODO: Path issues related to swift test of the output from a swiftbuild buildsystem
-                // let (stdout, stderr) = try sh(
-                //     swiftTest, "--package-path", packagePath, "--build-system", "swiftbuild"
-                // )
-                // #expect(!stderr.contains("error:"))
-                // #expect(stdout.contains("Test Suite 'All tests' passed"))
+                try withKnownIssue("""
+                       Linux: /lib/x86_64-linux-gnu/Scrt1.o:function _start: error: undefined reference to 'main'
+                       Windows: lld-link: error: subsystem must be defined
+                       MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found
+                    """) {
+                    try sh(swiftBuild, "--package-path", packagePath, "--build-system", buildSystemProvider.rawValue, "--vv")
+                    let (stdout, stderr) = try sh(
+                        swiftTest, "--package-path", packagePath, "--build-system", buildSystemProvider.rawValue, "--vv"
+                    )
+                    #expect(!stderr.contains("error:"))
+                    #expect(stdout.contains("Test Suite 'All tests' passed"))
+                } when: {
+                    buildSystemProvider == .swiftbuild
+                }
             }
         }
     }