wip: Implement Test Coverage Reporting

Author: maennchen

.github/workflows/ci.yml

@@ -30,6 +30,8 @@ jobs:
             deterministic: true
           - otp_version: "27.1"
             otp_latest: true
+          - otp_version: "27.1"
+            coverage: true
           - otp_version: "27.0"
           - otp_version: "26.0"
           - otp_version: master
@@ -58,9 +60,14 @@ jobs:
       - name: Erlang test suite
         run: make test_erlang
         continue-on-error: ${{ matrix.development }}
+        if: "${{ !matrix.coverage }}"
       - name: Elixir test suite
         run: make test_elixir
         continue-on-error: ${{ matrix.development }}
+        if: "${{ !matrix.coverage }}"
+      - name: "Calculate Coverage"
+        run: make cover
+        if: "${{ matrix.coverage }}"
       - name: Build docs (ExDoc main)
         if: ${{ matrix.otp_latest }}
         run: |
@@ -78,6 +85,12 @@ jobs:
           # Recompile System without .git
           cd lib/elixir && ../../bin/elixirc -o ebin lib/system.ex && cd -
           taskset 1 make check_reproducible
+      - name: "Upload Coverage Artifact"
+        if: "${{ matrix.coverage }}"
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: TestCoverage
+          path: cover/*
 
   test_windows:
     name: Windows Server 2019, Erlang/OTP ${{ matrix.otp_version }}

.gitignore

@@ -15,3 +15,4 @@
 /.eunit
 .elixir.plt
 erl_crash.dump
+/cover/

Makefile

@@ -53,6 +53,11 @@ lib/$(1)/ebin/Elixir.$(2).beam: $(wildcard lib/$(1)/lib/*.ex) $(wildcard lib/$(1
 test_$(1): test_formatted $(1)
 	@ echo "==> $(1) (ex_unit)"
 	$(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/$(TEST_FILES)";
+
+cover/exunit_$(1).coverdata:
+	$(Q) mkdir -p cover
+	$(Q) COVER_FILE="$(realpath cover)/exunit_$(1).coverdata" $(MAKE) test_$(1)
+cover/combined.coverdata: cover/exunit_$(1).coverdata
 endef
 
 define WRITE_SOURCE_DATE_EPOCH
@@ -175,6 +180,7 @@ clean: clean_man
 	rm -rf lib/mix/test/fixtures/git_sparse_repo/
 	rm -rf lib/mix/test/fixtures/archive/ebin/
 	rm -f erl_crash.dump
+	rm -rf cover/*
 
 clean_elixir:
 	$(Q) rm -f lib/*/ebin/Elixir.*.beam
@@ -287,6 +293,17 @@ test_stdlib: compile
 		cd lib/elixir && ../../bin/elixir --sname primary -r "test/elixir/test_helper.exs" -pr "test/elixir/**/$(TEST_FILES)"; \
 	fi
 
+cover/eunit_stdlib.coverdata:
+	$(Q) mkdir -p cover
+	$(Q) COVER_FILE="$(realpath cover)/eunit_stdlib.coverdata" $(MAKE) test_erlang
+cover/combined.coverdata: cover/eunit_stdlib.coverdata
+
+cover/combined.coverdata:
+	$(Q) bin/elixir ./cover.exs
+
+.PHONY: cover
+cover: cover/combined.coverdata
+
 #==> Dialyzer tasks
 
 DIALYZER_OPTS = --no_check_plt --fullpath -Werror_handling -Wunmatched_returns -Wunderspecs

cover.exs

@@ -0,0 +1,29 @@
+#!bin/elixir
+
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: 2021 The Elixir Team
+
+# TODO: Move this file in an acceptable location
+
+cover_dir = __ENV__.file |> Path.dirname() |> Path.join("cover")
+coverdata_inputs = cover_dir |> Path.join("{exunit,eunit}_*.coverdata") |> Path.wildcard()
+coverdata_output = Path.join(cover_dir, "combined.coverdata")
+ebins = __ENV__.file |> Path.dirname() |> Path.join("lib/*/ebin") |> Path.wildcard()
+
+_ = :cover.stop()
+{:ok, _pid} = :cover.start()
+
+for ebin <- ebins do
+  # TODO: Check result
+  :cover.compile_beam_directory(String.to_charlist(ebin))
+end
+
+for file <- coverdata_inputs do
+  :ok = :cover.import(String.to_charlist(file))
+end
+
+:ok = :cover.export(String.to_charlist(coverdata_output))
+
+{:ok, _} = Application.ensure_all_started(:mix)
+# TODO: How can I mute the "Analysis includes data from imported files" IO?
+:ok = Mix.Tasks.Test.Coverage.generate_cover_results(output: cover_dir)

cover_record.exs

@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: 2021 The Elixir Team
+
+# TODO: Move this file in an acceptable location
+
+case System.fetch_env("COVER_FILE") do
+  {:ok, file} ->
+    _ = :cover.stop()
+    {:ok, _pid} = :cover.start()
+
+    ebins = __ENV__.file |> Path.dirname() |> Path.join("lib/*/ebin") |> Path.wildcard()
+
+    for ebin <- ebins do
+      # TODO: Check result
+      :cover.compile_beam_directory(String.to_charlist(ebin))
+    end
+
+    System.at_exit(fn _status ->
+      :ok = :cover.export(String.to_charlist(file))
+    end)
+
+  :error ->
+    :ok
+end

lib/eex/test/test_helper.exs

@@ -5,6 +5,8 @@
 {line_exclude, line_include} =
   if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []}
 
+Code.eval_file("../../../../cover_record.exs", __ENV__.file)
+
 ExUnit.start(
   trace: !!System.get_env("TRACE"),
   include: line_include,

lib/elixir/test/elixir/test_helper.exs

@@ -120,6 +120,8 @@ source_exclude =
     []
   end
 
+Code.eval_file("../../../../../cover_record.exs", __ENV__.file)
+
 ExUnit.start(
   trace: !!System.get_env("TRACE"),
   assert_receive_timeout: assert_timeout,

lib/elixir/test/erlang/test_helper.erl

@@ -14,10 +14,13 @@
 
 test() ->
   application:ensure_all_started(elixir),
-  case eunit:test(?TESTS) of
-    error -> erlang:halt(1);
-    _Res  -> erlang:halt(0)
-  end.
+  enable_coverage_report(),
+  ExitCode = case eunit:test(?TESTS) of
+    error -> 1;
+    _Res  -> 0
+  end,
+  write_report_coverage(),
+  erlang:halt(ExitCode).
 
 % Execute a piece of code and purge given modules right after
 run_and_remove(Fun, Modules) ->
@@ -39,3 +42,20 @@ throw_erlang(String) ->
   {ok, Tokens, _} = erl_scan:string(String),
   {ok, [Form]} = erl_parse:parse_exprs(Tokens),
   erlang:error(io:format("~p~n", [Form])).
+
+enable_coverage_report() ->
+  case os:getenv("COVER_FILE") of
+    false -> ok;
+    _File ->
+      _ = cover:stop(),
+      {ok, _Pid} = cover:start(),
+      Ebin = filename:dirname(filename:absname(?FILE)) ++ "/../../ebin",
+      % TODO: Check Result
+      cover:compile_beam_directory(Ebin)
+  end.
+
+write_report_coverage() ->
+  case os:getenv("COVER_FILE") of
+    false -> ok;
+    File -> cover:export(File)
+  end.

lib/ex_unit/test/test_helper.exs

@@ -7,6 +7,8 @@ Logger.configure_backend(:console, colors: [enabled: false])
 {line_exclude, line_include} =
   if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []}
 
+Code.eval_file("../../../../cover_record.exs", __ENV__.file)
+
 ExUnit.start(
   trace: !!System.get_env("TRACE"),
   include: line_include,

lib/iex/test/test_helper.exs

@@ -26,6 +26,8 @@ source_exclude =
     []
   end
 
+Code.eval_file("../../../../cover_record.exs", __ENV__.file)
+
 ExUnit.start(
   assert_receive_timeout: assert_timeout,
   trace: !!System.get_env("TRACE"),

lib/logger/test/test_helper.exs

@@ -5,6 +5,8 @@
 {line_exclude, line_include} =
   if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []}
 
+Code.eval_file("../../../../cover_record.exs", __ENV__.file)
+
 ExUnit.start(
   trace: !!System.get_env("TRACE"),
   include: line_include,

lib/mix/lib/mix/tasks/test.coverage.ex

@@ -268,7 +268,8 @@ defmodule Mix.Tasks.Test.Coverage do
     end
   end
 
-  defp generate_cover_results(opts) do
+  @doc false
+  def generate_cover_results(opts) do
     {:result, ok, _fail} = :cover.analyse(:coverage, :line)
     ignore = opts[:ignore_modules] || []
     modules = Enum.reject(:cover.modules(), &ignored?(&1, ignore))

lib/mix/test/test_helper.exs

@@ -46,6 +46,8 @@ cover_exclude =
     []
   end
 
+Code.eval_file("../../../../cover_record.exs", __ENV__.file)
+
 ExUnit.start(
   trace: !!System.get_env("TRACE"),
   exclude: epmd_exclude ++ os_exclude ++ git_exclude ++ line_exclude ++ cover_exclude,