Skip reference tests on windows (#208)

* Normalize newlines in reference tests

* Revert "Normalize newlines in reference tests"

This reverts commit 331f8de.

* Skip reference tests on windows

* fixup! Skip reference tests on windows

* fixup paths

* fixup! fixup paths

* fixup! fixup! fixup paths

* fixup! fixup! fixup! fixup paths

* Window-proof more tests

* fixup! Window-proof more tests

* fixup! fixup! Window-proof more tests

* fixup! fixup! fixup! Window-proof more tests

* fixup! fixup! fixup! fixup! Window-proof more tests

* fixup! fixup! fixup! fixup! fixup! Window-proof more tests

* fixup! fixup! fixup! fixup! fixup! fixup! Window-proof more tests

Author: nickrobinson251

test/integrationtests.jl

@@ -90,7 +90,7 @@ end
 
     # warn if the path does not exist
     dne = joinpath(pkg, "does_not_exist")
-    dne_msg = "No such path \"$dne\""
+    dne_msg = "No such path $(repr(dne))"
     @test_logs (:warn, dne_msg) match_mode=:any begin
         runtests(dne)
     end
@@ -104,7 +104,7 @@ end
     # warn if the file is not a test file
     file = joinpath(pkg, "src", "foo.jl")
     @assert isfile(file)
-    file_msg = "\"$file\" is not a test file"
+    file_msg = "$(repr(file)) is not a test file"
     @test_logs (:warn, file_msg) match_mode=:any begin
         runtests(file)
     end
@@ -122,7 +122,7 @@ end
     # Warn for each invalid path and still run valid ones
     test_file = joinpath(pkg, "src", "foo_test.jl")
     @assert isfile(test_file)
-    results = @test_logs (:warn, "No such path \"$dne\"") (:warn, "\"$file\" is not a test file") match_mode=:any begin
+    results = @test_logs (:warn, "No such path $(repr(dne))") (:warn, "$(repr(file)) is not a test file") match_mode=:any begin
         encased_testset() do
             runtests(test_file, dne, file)
         end
@@ -400,6 +400,7 @@ end
         Test.print_test_results(testset)
     end
     # Test with `contains` rather than `match` so failure print an informative message.
+    if !Base.Sys.iswindows() # so we can hardcode filepaths to keep the test readable
     @test contains(
         c.output,
         r"""
@@ -427,6 +428,7 @@ end
               foo                         \|    2      2  \s*\d*.\ds
         """
     )
+    end
     # verbose_results=false
     testset = with_test_package("TestsInSrc.jl") do
         runtests(verbose_results=false)
@@ -621,29 +623,38 @@ end
 end
 
 @testset "Warn on empty test set -- integration test" begin
+    fullpath = joinpath(TEST_FILES_DIR, "_empty_testsets_tests.jl")
+    relfpath = relpath(fullpath, pkgdir(ReTestItems))
     @test_logs (:warn, """
-    Test item "Warn on empty test set -- integration test" at test/testfiles/_empty_testsets_tests.jl:1 contains test sets without tests:
+    Test item "Warn on empty test set -- integration test" at $relfpath:1 contains test sets without tests:
     "Empty testset"
     "Inner empty testset"
     """) match_mode=:any begin
-        ReTestItems.runtests(joinpath(TEST_FILES_DIR, "_empty_testsets_tests.jl"))
+        ReTestItems.runtests(fullpath)
     end
 end
 
 @testset "log capture for an errored TestSetup" begin
+    path = joinpath("test", "error_in_setup_test.jl")
     c = IOCapture.capture() do
         results = with_test_package("DontPass.jl") do
-            runtests("test/error_in_setup_test.jl"; nworkers=1)
+            runtests(path; nworkers=1)
         end
     end
+if Base.Sys.iswindows()
+    @test occursin(
+        "\e[36m\e[1mCaptured logs\e[22m\e[39m for test setup \"SetupThatErrors\" (dependency of \"bad setup, good test\") at",
+        replace(c.output, r" on worker \d+" => "")
+    )
+else
     @test occursin("""
-    \e[36m\e[1mCaptured logs\e[22m\e[39m for test setup \"SetupThatErrors\" (dependency of \"bad setup, good test\") at \e[39m\e[1mtest/error_in_setup_test.jl:1\e[22m
+    \e[36m\e[1mCaptured logs\e[22m\e[39m for test setup \"SetupThatErrors\" (dependency of \"bad setup, good test\") at \e[39m\e[1m$(path):1\e[22m
     SetupThatErrors msg
     """,
     replace(c.output, r" on worker \d+" => ""))
 
     @test occursin("""
-    \e[36m\e[1mCaptured logs\e[22m\e[39m for test setup \"SetupThatErrors\" (dependency of \"bad setup, bad test\") at \e[39m\e[1mtest/error_in_setup_test.jl:1\e[22m
+    \e[36m\e[1mCaptured logs\e[22m\e[39m for test setup \"SetupThatErrors\" (dependency of \"bad setup, bad test\") at \e[39m\e[1m$(path):1\e[22m
     SetupThatErrors msg
     """,
     replace(c.output, r" on worker \d+" => ""))
@@ -652,19 +663,20 @@ end
     # that we don't accumulate logs from all previous failed attempts (which would get
     # really spammy if the test setup is used by 100 test items).
     @test !occursin("""
-        \e[36m\e[1mCaptured logs\e[22m\e[39m for test setup \"SetupThatErrors\" (dependency of \"bad setup, good test\") at \e[39m\e[1mtest/error_in_setup_test.jl:1\e[22m
+        \e[36m\e[1mCaptured logs\e[22m\e[39m for test setup \"SetupThatErrors\" (dependency of \"bad setup, good test\") at \e[39m\e[1m$(path):1\e[22m
         SetupThatErrors msg
         SetupThatErrors msg
         """,
         replace(c.output, r" on worker \d+" => "")
     )
     @test !occursin("""
-        \e[36m\e[1mCaptured logs\e[22m\e[39m for test setup \"SetupThatErrors\" (dependency of \"bad setup, bad test\") at \e[39m\e[1mtest/error_in_setup_test.jl:1\e[22m
+        \e[36m\e[1mCaptured logs\e[22m\e[39m for test setup \"SetupThatErrors\" (dependency of \"bad setup, bad test\") at \e[39m\e[1m$(path):1\e[22m
         SetupThatErrors msg
         SetupThatErrors msg
         """,
         replace(c.output, r" on worker \d+" => "")
     )
+end # iswindows
 end
 
 @testset "test crashing testitem" begin
@@ -685,7 +697,8 @@ end
     # Test the error is as expected
     err = only(non_passes(results))
     @test err.test_type == :nontest_error
-    @test err.value == string(ErrorException("Worker process aborted (signal=6) running test item \"Abort\" (run=1)"))
+    sig = Base.Sys.iswindows() ? 0 : 6
+    @test err.value == string(ErrorException("Worker process aborted (signal=$(sig)) running test item \"Abort\" (run=1)"))
 end
 
 @testset "test retrying failing testitem" begin
@@ -908,7 +921,12 @@ end
 
 @testset "Duplicate names in same file throws" begin
     file = joinpath(TEST_FILES_DIR, "_duplicate_names_test.jl")
-    expected_msg = Regex("Duplicate test item name `dup` in file `test/testfiles/_duplicate_names_test.jl` at line 4")
+    relfpath = relpath(file, pkgdir(ReTestItems))
+    expected_msg = if Base.Sys.iswindows()
+        Regex("Duplicate test item name `dup` in file")
+    else
+        Regex("Duplicate test item name `dup` in file `$(relfpath)` at line 4")
+    end
     @test_throws expected_msg runtests(file; nworkers=0)
     @test_throws expected_msg runtests(file; nworkers=1)
 end
@@ -967,51 +985,58 @@ end
         return logs
     end
 
-    @testset "timeout_profile_wait=0 means no CPU profile" begin
-    capture_timeout_profile(0) do logs
-        @test !occursin("Information request received", logs)
+    if Base.Sys.iswindows()
+        @testset "Windows not supported" begin
+            capture_timeout_profile(1) do logs
+                @test occursin("CPU profiles on timeout is not supported on Windows, ignoring `timeout_profile_wait`", logs)
+            end
+        end
+    else
+        @testset "timeout_profile_wait=0 means no CPU profile" begin
+            capture_timeout_profile(0) do logs
+                @test !occursin("Information request received", logs)
+            end
         end
-    end
-
 
-    default_peektime = Profile.get_peek_duration()
-    @testset "non-zero timeout_profile_wait means we collect a CPU profile" begin
-    capture_timeout_profile(5) do logs
-        @test occursin("Information request received. A stacktrace will print followed by a $(default_peektime) second profile", logs)
-            @test count(r"pthread_cond_wait|__psych_cvwait", logs) > 0 # the stacktrace was printed (will fail on Windows)
-        @test occursin("Profile collected.", logs)
+        default_peektime = Profile.get_peek_duration()
+        @testset "non-zero timeout_profile_wait means we collect a CPU profile" begin
+            capture_timeout_profile(5) do logs
+                @test occursin("Information request received. A stacktrace will print followed by a $(default_peektime) second profile", logs)
+                @test count(r"pthread_cond_wait|__psych_cvwait", logs) > 0 # the stacktrace was printed (will fail on Windows)
+                @test occursin("Profile collected.", logs)
+            end
         end
-    end
 
 
-    @testset "`set_peek_duration` is respected in `worker_init_expr`" begin
-    capture_timeout_profile(5, worker_init_expr=:(using Profile; Profile.set_peek_duration($default_peektime + 1.0))) do logs
-        @test occursin("Information request received. A stacktrace will print followed by a $(default_peektime + 1.0) second profile", logs)
-            @test count(r"pthread_cond_wait|__psych_cvwait", logs) > 0 # the stacktrace was printed (will fail on Windows)
-        @test occursin("Profile collected.", logs)
+        @testset "`set_peek_duration` is respected in `worker_init_expr`" begin
+            capture_timeout_profile(5, worker_init_expr=:(using Profile; Profile.set_peek_duration($default_peektime + 1.0))) do logs
+                @test occursin("Information request received. A stacktrace will print followed by a $(default_peektime + 1.0) second profile", logs)
+                @test count(r"pthread_cond_wait|__psych_cvwait", logs) > 0 # the stacktrace was printed (will fail on Windows)
+                @test occursin("Profile collected.", logs)
+            end
         end
-    end
 
 
-    # The RETESTITEMS_TIMEOUT_PROFILE_WAIT environment variable can be used to set the timeout_profile_wait.
-    @testset "RETESTITEMS_TIMEOUT_PROFILE_WAIT environment variable" begin
-    withenv("RETESTITEMS_TIMEOUT_PROFILE_WAIT" => "5") do
-        capture_timeout_profile(nothing) do logs
-            @test occursin("Information request received", logs)
-                @test count(r"pthread_cond_wait|__psych_cvwait", logs) > 0 # the stacktrace was printed (will fail on Windows)
-            @test occursin("Profile collected.", logs)
+        # The RETESTITEMS_TIMEOUT_PROFILE_WAIT environment variable can be used to set the timeout_profile_wait.
+        @testset "RETESTITEMS_TIMEOUT_PROFILE_WAIT environment variable" begin
+            withenv("RETESTITEMS_TIMEOUT_PROFILE_WAIT" => "5") do
+                capture_timeout_profile(nothing) do logs
+                    @test occursin("Information request received", logs)
+                    @test count(r"pthread_cond_wait|__psych_cvwait", logs) > 0 # the stacktrace was printed (will fail on Windows)
+                    @test occursin("Profile collected.", logs)
+                end
             end
         end
-    end
 
-    # The profile is collected for each worker thread.
-    @testset "CPU profile with $(repr(log_capture))" for log_capture in (:eager, :batched)
-        capture_timeout_profile(5, nworker_threads=VERSION >= v"1.9" ? "3,2" : "3", logs=log_capture) do logs
-        @test occursin("Information request received", logs)
-            @test count(r"pthread_cond_wait|__psych_cvwait", logs) > 0 # the stacktrace was printed (will fail on Windows)
-        @test occursin("Profile collected.", logs)
+        # The profile is collected for each worker thread.
+        @testset "CPU profile with $(repr(log_capture))" for log_capture in (:eager, :batched)
+            capture_timeout_profile(5, nworker_threads=VERSION >= v"1.9" ? "3,2" : "3", logs=log_capture) do logs
+                @test occursin("Information request received", logs)
+                @test count(r"pthread_cond_wait|__psych_cvwait", logs) > 0 # the stacktrace was printed (will fail on Windows)
+                @test occursin("Profile collected.", logs)
+            end
         end
-    end
+    end # iswindows
 end
 
 @testset "worker always crashes immediately" begin
@@ -1334,7 +1359,8 @@ end
         :crash   => "_failfast_crash_tests.jl",
     )
         testitem_timeout = 5
-        file = joinpath(TEST_FILES_DIR, filename)
+        fullpath = joinpath(TEST_FILES_DIR, filename)
+        relfpath = relpath(fullpath, pkgdir(ReTestItems))
         # For 0 or 1 workers, we expect to fail on the second testitem out of 3.
         # If running with 3 workers, then all 3 testitems will be running in parallel,
         # so we expect to see all 3 testitems run, even though one fails.
@@ -1346,7 +1372,7 @@ end
                 continue
             end
             c = IOCapture.capture() do
-                encased_testset(() -> runtests(file; nworkers, testitem_timeout, retries=1, failfast=true))
+                encased_testset(() -> runtests(fullpath; nworkers, testitem_timeout, retries=1, failfast=true))
             end
             results = c.value
             if nworkers == 3
@@ -1359,7 +1385,7 @@ end
             # @show c.output
             @test contains(c.output, "Retrying")  # check retries are happening
             @test count(r"\[ Fail Fast:", c.output) == 2
-            msg = "[ Fail Fast: Test item \"bad\" at test/testfiles/$filename:4 failed. Cancelling tests."
+            msg = "[ Fail Fast: Test item \"bad\" at $relfpath:4 failed. Cancelling tests."
             @test contains(c.output, msg)
             if nworkers == 3
                 @test contains(c.output, "[ Fail Fast: 3/3 test items were run.")

test/internals.jl

@@ -169,7 +169,8 @@ end # `include_testfiles!` testset
 @testset "report_empty_testsets" begin
     using ReTestItems: TestItem, report_empty_testsets, PerfStats, ScheduledForEvaluation
     using Test: DefaultTestSet, Fail, Error
-    ti = TestItem(Ref(42), "Dummy TestItem", "DummyID", [], false, [], 0, nothing, false, nothing, "source/path", 42, ".", nothing)
+    path = joinpath("source", "path")
+    ti = TestItem(Ref(42), "Dummy TestItem", "DummyID", [], false, [], 0, nothing, false, nothing, path, 42, ".", nothing)
 
     ts = DefaultTestSet("Empty testset")
     report_empty_testsets(ti, ts)
@@ -179,7 +180,7 @@ end # `include_testfiles!` testset
     push!(ts.results, DefaultTestSet("Empty testset"))
     # Only the inner testset is considered empty
     @test_logs (:warn, """
-        Test item "Dummy TestItem" at source/path:42 contains test sets without tests:
+        Test item "Dummy TestItem" at $(path):42 contains test sets without tests:
         "Empty testset"
         """) begin
         report_empty_testsets(ti, ts)
@@ -190,7 +191,7 @@ end # `include_testfiles!` testset
     push!(ts.results, DefaultTestSet("Empty testset 2"))
     # Only the inner testsets are considered empty
     @test_logs (:warn, """
-        Test item "Dummy TestItem" at source/path:42 contains test sets without tests:
+        Test item "Dummy TestItem" at $(path):42 contains test sets without tests:
         "Empty testset 1"
         "Empty testset 2"
         """) begin
@@ -277,8 +278,8 @@ end
     @assert !ReTestItems.is_test_file(nontest_file)
     @assert !ReTestItems.is_testsetup_file(nontest_file)
     @test _validated_paths((nontest_file,), false) == ()
-    @test_logs (:warn, "\"$nontest_file\" is not a test file") _validated_paths((nontest_file,), false)
-    @test_throws ArgumentError("\"$nontest_file\" is not a test file") _validated_paths((nontest_file,), true)
+    @test_logs (:warn, "$(repr(nontest_file)) is not a test file") _validated_paths((nontest_file,), false)
+    @test_throws ArgumentError("$(repr(nontest_file)) is not a test file") _validated_paths((nontest_file,), true)
 end
 
 @testset "skiptestitem" begin
@@ -383,25 +384,27 @@ end
 
 @testset "nestedrelpath" begin
     using ReTestItems: nestedrelpath
-    @assert Base.Filesystem.path_separator == "/"
-    path = "test/dir/foo_test.jl"
-    @test nestedrelpath(path, "test")  == relpath(path, "test")  == "dir/foo_test.jl"
-    @test nestedrelpath(path, "test/") == relpath(path, "test/") == "dir/foo_test.jl"
-    @test nestedrelpath(path, "test/dir")  == relpath(path, "test/dir")  == "foo_test.jl"
-    @test nestedrelpath(path, "test/dir/") == relpath(path, "test/dir/") == "foo_test.jl"
-    @test nestedrelpath(path, "test/dir/foo_test.jl") == relpath(path, "test/dir/foo_test.jl") == "."
-
-    # unlike `relpath`: if `startdir` is not a prefix of `path`, the assumption is violated,
-    # and `path` is just returned as-is
-    @test nestedrelpath(path, "test/dir/foo_") == "test/dir/foo_test.jl"
-    @test nestedrelpath(path, "test/dir/other") == "test/dir/foo_test.jl"
-    @test nestedrelpath(path, "test/dir/other/bar_test.jl") == "test/dir/foo_test.jl"
-
-    # leading '/' doesn't get ignored or stripped
-    @test nestedrelpath("/a/b/c", "/a/b") == "c"
-    @test nestedrelpath("/a/b/c", "a/b") == "/a/b/c"
-    @test nestedrelpath("/a/b", "/a/b/c") == "/a/b"
-    @test nestedrelpath("/a/b", "c") == "/a/b"
+    if !Base.Sys.iswindows()
+        @assert Base.Filesystem.path_separator == "/"
+        path = "test/dir/foo_test.jl"
+        @test nestedrelpath(path, "test")  == relpath(path, "test")  == "dir/foo_test.jl"
+        @test nestedrelpath(path, "test/") == relpath(path, "test/") == "dir/foo_test.jl"
+        @test nestedrelpath(path, "test/dir")  == relpath(path, "test/dir")  == "foo_test.jl"
+        @test nestedrelpath(path, "test/dir/") == relpath(path, "test/dir/") == "foo_test.jl"
+        @test nestedrelpath(path, "test/dir/foo_test.jl") == relpath(path, "test/dir/foo_test.jl") == "."
+
+        # unlike `relpath`: if `startdir` is not a prefix of `path`, the assumption is violated,
+        # and `path` is just returned as-is
+        @test nestedrelpath(path, "test/dir/foo_") == "test/dir/foo_test.jl"
+        @test nestedrelpath(path, "test/dir/other") == "test/dir/foo_test.jl"
+        @test nestedrelpath(path, "test/dir/other/bar_test.jl") == "test/dir/foo_test.jl"
+
+        # leading '/' doesn't get ignored or stripped
+        @test nestedrelpath("/a/b/c", "/a/b") == "c"
+        @test nestedrelpath("/a/b/c", "a/b") == "/a/b/c"
+        @test nestedrelpath("/a/b", "/a/b/c") == "/a/b"
+        @test nestedrelpath("/a/b", "c") == "/a/b"
+    end
 end
 
 end # internals.jl testset

test/macros.jl

@@ -230,7 +230,7 @@ end
 @testset "testitem `_id` keyword" begin
     # Should default to `repr(hash(name, hash(file)))` where `file` is relative to the root
     # of the project being tested.
-    file = "test/macros.jl" # this file
+    file = joinpath("test, macros.jl") # this file
     # set the source to be this file, so that the test is valid even when run in the REPL.
     ti1 = @testitem "one" _run=false _source=LineNumberNode(@__LINE__, file) begin; end;
     @test ti1.id == repr(hash("one", hash(file)))
@@ -247,7 +247,7 @@ end
     expected = VERSION < v"1.8" ? LoadError : "must be passed a string"
     @test_throws expected (@eval @testitem("four", _id=1, _run=false, begin end))
     # Cannot detect type of `id` at macro-expansion time, so throws run-time error
-    expected = VERSION < v"1.8" ? MethodError : "MethodError: Cannot `convert` an object of type UInt64 to an object of type String"
+    expected = VERSION < v"1.8" ? MethodError : "MethodError: Cannot `convert` an object of type $(UInt) to an object of type String"
     @test_throws expected (@eval @testitem("five", _id=hash("five"), _run=false, begin end))
 end
 

test/runtests.jl

@@ -20,13 +20,18 @@ end
         include("macros.jl")
         include("integrationtests.jl")
         include("log_capture.jl")
-        # junit_xml reference tests are very sensitive to changes in text output which is not
-        # stable across minor Julia versions, i.e. we expect these to fail on upcoming Julia
-        # releases so we may as well skip them (so PkgEval doesn't always fail for us).
-        if isempty(VERSION.prerelease)
-            include("junit_xml.jl")
+        @static if Base.Sys.iswindows()
+            # https://github.com/JuliaTesting/ReTestItems.jl/issues/209
+            @warn "Skipping JUnit XML reference tests on windows"
         else
-            @warn "Skipping JUnit XML reference tests on unrelease Julia version" VERSION
+            # junit_xml reference tests are very sensitive to changes in text output which is not
+            # stable across minor Julia versions, i.e. we expect these to fail on upcoming Julia
+            # releases so we may as well skip them (so PkgEval doesn't always fail for us).
+            if !isempty(VERSION.prerelease)
+                @warn "Skipping JUnit XML reference tests on unrelease Julia version" VERSION
+            else
+                include("junit_xml.jl")
+            end
         end
     end