Speed up tests

- Don't activate test package environment
- Stop checking registry as much
- Remove tests that duplicate testing
- Simplify all test packages
- Don't clone a fresh registry - we're not testing it

Author: Mal Miller

Project.toml

@@ -1,6 +1,6 @@
 name = "TestReports"
 uuid = "dcd651b4-b50a-5b6b-8f22-87e9f253a252"
-version = "0.5.2"
+version = "0.5.3"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

test/recordproperty.jl

@@ -66,8 +66,7 @@ using TestReports
         # Test full packaage
         pkg = "TestsWithProperties"
         temp_pkg_dir() do tmp
-            copy_test_package(tmp, pkg)
-            Pkg.activate(joinpath(tmp, pkg))
+            Pkg.develop(Pkg.PackageSpec(path=test_package_path(pkg)))
             TestReports.test(pkg)
         end
         logfile = joinpath(@__DIR__, "testlog.xml")

test/references/test_with_properties.txt

@@ -1,2 +1,2 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites tests="9" failures="0" errors="0"><testsuite name="Math/Multiplication" tests="2" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/Multiplication" time="0.0"/><testcase name="pass (info lost) (Test 2)" id="_testcase_id_" classname="Math/Multiplication" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="ID" value="1"/></properties></testsuite><testsuite name="Math/addition/negative addition" tests="2" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/addition/negative addition" time="0.0"/><testcase name="pass (info lost) (Test 2)" id="_testcase_id_" classname="Math/addition/negative addition" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="AdditionalNest" value="true"/><property name="ID" value="2"/></properties></testsuite><testsuite name="Math/addition" tests="2" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/addition" time="0.0"/><testcase name="pass (info lost) (Test 2)" id="_testcase_id_" classname="Math/addition" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="ID" value="2"/></properties></testsuite><testsuite name="Math" tests="2" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math" time="0.0"/><testcase name="pass (info lost) (Test 2)" id="_testcase_id_" classname="Math" time="0.0"/><properties><property name="File" value="runtests.jl"/></properties></testsuite><testsuite name="Types" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="4"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Types" time="0.0"/><properties><property name="Float" value="0.5"/><property name="Int" value="1"/><property name="String" value="TextTests"/><property name="List" value="[&quot;1&quot;]"/><property name="Symbol" value="asymbol"/></properties></testsuite></testsuites>
+<testsuites tests="5" failures="0" errors="0"><testsuite name="Outer/Middle 1" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Outer/Middle 1" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="ID" value="1"/></properties></testsuite><testsuite name="Outer/Middle 2/Inner" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Outer/Middle 2/Inner" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="AdditionalNest" value="true"/><property name="ID" value="2"/></properties></testsuite><testsuite name="Outer/Middle 2" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Outer/Middle 2" time="0.0"/><properties><property name="File" value="runtests.jl"/><property name="ID" value="2"/></properties></testsuite><testsuite name="Outer" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Outer" time="0.0"/><properties><property name="File" value="runtests.jl"/></properties></testsuite><testsuite name="Types" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="4"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Types" time="0.0"/><properties><property name="Float" value="0.5"/><property name="Int" value="1"/><property name="String" value="TextTests"/><property name="List" value="[&quot;1&quot;]"/><property name="Symbol" value="asymbol"/></properties></testsuite></testsuites>

test/reportgeneration.jl

@@ -16,24 +16,25 @@ end
 
 @testset "Runner tests" begin
     @testset "Installed packages by name in primary environment" begin
-        # Pkg.add
+        # For speed avoid updating registry, we have a fresh one anyway
+        Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true
+        # Pkg.add test
         Pkg.add(TEST_PKG.name)
         test_successful_testrun(() -> TestReports.test(TEST_PKG.name), TEST_PKG.name)
         Pkg.rm(TEST_PKG.name)
         @test_throws TestReports.PkgTestError TestReports.test(TEST_PKG.name)
-
-        # Pkg.develop
-        pkgname = "PassingTests"
-        Pkg.develop(Pkg.PackageSpec(path=joinpath(@__DIR__, "test_packages", pkgname)))
-        test_successful_testrun(() -> TestReports.test(pkgname), pkgname)
-        Pkg.rm("PassingTests")
     end
 
     @testset "Activated projects - TestReports in stacked environment" begin
         @testset "by name" begin
             # The test run should not fail when passed the name of the project
             # that is activated and fail when its deactivated
-            test_active_package_expected_pass("PassingTests")
+            temp_pkg_dir() do tmp
+                pkg = "PassingTests"
+                path = copy_test_package(tmp, pkg)
+                Pkg.activate(path)
+                test_successful_testrun(() -> TestReports.test(pkg), pkg)
+            end
             @test_throws TestReports.PkgTestError TestReports.test(TEST_PKG.name)
         end
 
@@ -57,45 +58,40 @@ end
 
     @testset "Non-package error in runner" begin
         temp_pkg_dir() do tmp
-            pkgname = "PassingTests"
-            Pkg.develop(Pkg.PackageSpec(path=joinpath(@__DIR__, "test_packages", pkgname)))
+            pkg = "PassingTests"
+            Pkg.develop(Pkg.PackageSpec(path=test_package_path(pkg)))
             # Pass non-existing argument to julia to make run command fail
-            @test_throws TestReports.PkgTestError TestReports.test(pkgname, julia_args=`--doesnt-exist`)
+            @test_throws TestReports.PkgTestError TestReports.test(pkg, julia_args=`--doesnt-exist`)
         end
     end
 end
 
 @testset "Test packages" begin
     # Errors
-    test_active_package_expected_fail("FailedTest")
-    test_active_package_expected_fail("ErroredTest")
-    test_active_package_expected_fail("NoTestFile")
+    test_package_expected_fail("FailedTest")
+    test_package_expected_fail("ErroredTest")
+    test_package_expected_fail("NoTestFile")
 
     # Various test dependencies
     test_pkgs = [
         "TestsWithDeps",
         "TestsWithTestDeps"
     ]
-    for pkg in test_pkgs
-        test_active_package_expected_pass(pkg)
-    end
+    foreach(test_package_expected_pass, test_pkgs)
 
     # Test file project file tests, 1.2 and above
     @static if VERSION >= v"1.2.0"
         test_pkgs = [
             "TestsWithProjectFile",
             "TestsWithProjectFileWithTestDeps"
         ]
-        for pkg in test_pkgs
-            test_active_package_expected_pass(pkg)
-        end
+        foreach(test_package_expected_pass, test_pkgs)
     end
 
     # Test arguments
     temp_pkg_dir() do tmp
         pkg = "TestArguments"
-        copy_test_package(tmp, pkg)
-        Pkg.activate(joinpath(tmp, pkg))
+        Pkg.develop(Pkg.PackageSpec(path=test_package_path(pkg)))
         test_successful_testrun(() -> TestReports.test(pkg; test_args=`a b`, julia_args=`--quiet --check-bounds=no`), pkg)
         test_successful_testrun(() -> TestReports.test(pkg; test_args=["a", "b"], julia_args=`--quiet --check-bounds=no`), pkg)
     end
@@ -107,27 +103,21 @@ end
     # Single package tests
     TestReports.test(TEST_PKG.name)
     @test isfile(joinpath(pwd(),"testlog.xml"))
-    TestReports.test(TEST_PKG.name; logfilename="changedname.xml")
-    @test isfile(joinpath(pwd(),"changedname.xml"))
     new_path = joinpath(pwd(), "NonExistentDir")
-    TestReports.test(TEST_PKG.name; logfilename="testlog.xml", logfilepath=new_path)
-    @test isfile(joinpath(new_path,"testlog.xml"))
+    TestReports.test(TEST_PKG.name; logfilename="changedname.xml", logfilepath=new_path)
+    @test isfile(joinpath(new_path,"changedname.xml"))
     Pkg.rm(TEST_PKG.name)
 
     # Multiple package test
     temp_pkg_dir() do tmp
-        copy_test_package(tmp, "PassingTests")
-        Pkg.activate(tmp)
-        Pkg.develop(Pkg.PackageSpec(path=joinpath(tmp, "PassingTests")))
+        Pkg.develop(Pkg.PackageSpec(path=test_package_path("PassingTests")))
         Pkg.add(TEST_PKG.name)
         TestReports.test([TEST_PKG.name, "PassingTests"])
         @test isfile(joinpath(pwd(),"Example_testlog.xml"))
         @test isfile(joinpath(pwd(),"PassingTests_testlog.xml"))
         TestReports.test([TEST_PKG.name, "PassingTests"]; logfilename=["testlog1.xml", "testlog2.xml"])
         @test isfile(joinpath(pwd(),"testlog1.xml"))
         @test isfile(joinpath(pwd(),"testlog2.xml"))
-        Pkg.rm("PassingTests")
-        Pkg.rm(TEST_PKG.name)
     end
 
     # Errors
@@ -136,7 +126,7 @@ end
     @test_throws TypeError TestReports.test(TEST_PKG.name; logfilename=["ThisShouldJustBeAString.xml"])
 
     # Tidy up
-    rm.(joinpath.(Ref(pwd()), ["testlog.xml", "changedname.xml", "Example_testlog.xml", "PassingTests_testlog.xml", "testlog1.xml", "testlog2.xml"]))
+    rm.(joinpath.(Ref(pwd()), ["testlog.xml", "Example_testlog.xml", "PassingTests_testlog.xml", "testlog1.xml", "testlog2.xml"]))
     rm(new_path, recursive=true)
 end
 

test/runtests.jl

@@ -11,6 +11,3 @@ include("utils.jl")
     @testset "report generation" begin include("reportgeneration.jl") end
     @testset "runner internals" begin include("runnerinternals.jl") end
 end
-
-# clean up locally cached registry
-rm(joinpath(@__DIR__, "registries"); force=true, recursive=true)

test/test_packages/CustomTestSet/src/CustomTestSet.jl

@@ -1,5 +1,3 @@
 module CustomTestSet
 
-greet() = print("Hello World!")
-
 end # module

test/test_packages/CustomTestSet/test/runtests.jl

@@ -42,29 +42,4 @@ end
         @test true
     end
     @test true
-end
-
-# Own time_taken field
-mutable struct TimeTakenTestSet <: AbstractTestSet
-    description::String
-    results::Vector
-    time_taken::Int64
-end
-TimeTakenTestSet(desc) = TimeTakenTestSet(desc, [], -1)
-record(ts::TimeTakenTestSet, t) = (push!(ts.results, t); t)
-function finish(ts::TimeTakenTestSet)
-    # If we are a nested test set, do not print a full summary
-    # now - let the parent test set do the printing
-    if get_testset_depth() != 0
-        # Attach this test set to the parent test set
-        parent_ts = get_testset()
-        record(parent_ts, ts)
-        return ts
-    end
-
-    return ts
-end
-
-@testset TimeTakenTestSet "ts6" begin
-    @test true
 end

test/test_packages/DefaultTestSet/Project.toml

@@ -1,7 +1,3 @@
 module ErroredTest
 
-export simple_sum
-
-simple_sum(a, b) = a + b
-
 end # module

test/test_packages/DefaultTestSet/src/DefaultTestSet.jl

@@ -1,4 +1,3 @@
 using ErroredTest, Test
 
-# Function doesn't exist, will error
-@test function_doesnt_exist(1, 2)
+@test error("ErroredTest error")

test/test_packages/DefaultTestSet/test/runtests.jl

@@ -1,7 +1,3 @@
 module FailedTest
 
-export simple_sum
-
-simple_sum(a, b) = a + b
-
 end # module

test/test_packages/ErroredTest/src/ErroredTest.jl

@@ -1,4 +1,3 @@
 using FailedTest, Test
 
-# Result will be 3, test will fail
-@test simple_sum(1, 2) == 10
+@test false

test/test_packages/ErroredTest/test/runtests.jl

@@ -1,5 +1,3 @@
 module NoDescriptionCustomTestSet
 
-greet() = print("Hello World!")
-
 end # module

test/test_packages/FailedTest/src/FailedTest.jl

@@ -1,5 +1,3 @@
 module NoResultsCustomTestSet
 
-greet() = print("Hello World!")
-
 end # module

test/test_packages/FailedTest/test/runtests.jl

@@ -1,5 +1,3 @@
 module NoTestFile
 
-greet() = print("Hello World!")
-
 end # module

test/test_packages/NoDescriptionCustomTestSet/src/NoDescriptionCustomTestSet.jl

@@ -1,7 +1,3 @@
 module PassingTests
 
-export simple_sum
-
-simple_sum(a, b) = a + b
-
 end # module

test/test_packages/NoResultsCustomTestSet/src/NoResultsCustomTestSet.jl

@@ -1,3 +1,3 @@
 using PassingTests, Test
 
-@test simple_sum(1, 2) == 3
+@test true

test/test_packages/NoTestFile/src/NoTestFile.jl

@@ -3,7 +3,7 @@ uuid = "3ef0d59a-7f5b-48e9-8833-7baa3598d41d"
 version = "0.1.0"
 
 [deps]
-Dictionaries = "85a47980-9c8c-11e8-2b9f-f7ca1fa99fb4"
+Example = "7876af07-990d-54b4-ab0e-23690620f79a"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/test_packages/PassingTests/src/PassingTests.jl

@@ -1,9 +1,5 @@
 module TestsWithDeps
 
-using Dictionaries
-
-export create_hash_dict
-
-create_hash_dict(a, b) = Dictionary(["key1", "key2"], [a, b])
+using Example
 
 end # module

test/test_packages/PassingTests/test/runtests.jl

@@ -1,3 +1 @@
-using TestsWithDeps, Dictionaries, Test
-
-@test create_hash_dict(1, 2) == Dictionary(["key1", "key2"], [1, 2])
+using TestsWithDeps, Test

test/test_packages/TestsWithDeps/Project.toml

@@ -1,7 +1,3 @@
 module TestsWithProjectFile
 
-export simple_sum
-
-simple_sum(a, b) = a + b
-
 end # module

test/test_packages/TestsWithDeps/src/TestsWithDeps.jl

@@ -1,3 +1,3 @@
 using TestsWithProjectFile, Test
 
-@test simple_sum(1, 2) == 3
+@test true

test/test_packages/TestsWithDeps/test/runtests.jl

@@ -1,7 +1,3 @@
 module TestsWithProjectFileWithTestDeps
 
-export simple_sum
-
-simple_sum(a, b) = a + b
-
 end # module

test/test_packages/TestsWithProjectFile/src/TestsWithProjectFile.jl

@@ -1,3 +1,3 @@
 [deps]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-Dictionaries = "85a47980-9c8c-11e8-2b9f-f7ca1fa99fb4"
+Example = "7876af07-990d-54b4-ab0e-23690620f79a"

test/test_packages/TestsWithProjectFile/test/runtests.jl

@@ -1,4 +1 @@
-using TestsWithProjectFileWithTestDeps, Dictionaries, Test
-
-results = Dictionary(["result1"], [3])
-@test simple_sum(1, 2) == results["result1"]
+using TestsWithProjectFileWithTestDeps, Example, Test

test/test_packages/TestsWithProjectFileWithTestDeps/src/TestsWithProjectFileWithTestDeps.jl

@@ -2,27 +2,23 @@ using Test
 using TestReports
 using Base.Threads
 
-@testset "Math" begin
+@testset "Outer" begin
     recordproperty("File", "runtests.jl")
 
-    @testset "Multiplication" begin
+    @testset "Middle 1" begin
         recordproperty("ID", 1)
-        @test 1*3 == 3
-        @test 1*4 == 4
+        @test true
     end
 
-    @test 4 % 2 == 0
-    @test 16 == 16
+    @test true
 
-    @testset "addition" begin
+    @testset "Middle 2" begin
         recordproperty("ID", 2)
-        @test 1+1 == 2
-        @test 1+4 == 5
+        @test true
 
-        @testset "negative addition" begin
+        @testset "Inner" begin
             recordproperty("AdditionalNest", true)
-            @test 1 + -1 == 0
-            @test 10 + -5 == 5
+            @test true
         end
     end
 end
@@ -33,5 +29,5 @@ end
     recordproperty("Float", 0.5) 
     recordproperty("List", ["1"]) 
     recordproperty("Symbol", :asymbol) 
-    @test occursin("i", "in")
+    @test true
 end