Implement Test Coverage Reporting

Author: maennchen

.github/workflows/ci.yml

@@ -21,13 +21,16 @@ permissions:
 
 jobs:
   test_linux:
-    name: Ubuntu 24.04, Erlang/OTP ${{ matrix.otp_version }}${{ matrix.deterministic && ' (deterministic)' || '' }}
+    name: Ubuntu 24.04, Erlang/OTP ${{ matrix.otp_version }}${{ matrix.deterministic && ' (deterministic)' || '' }}${{ matrix.coverage && ' (coverage)' || '' }}
     strategy:
       fail-fast: false
       matrix:
         include:
           - otp_version: "27.1"
             deterministic: true
+          - otp_version: "27.1"
+            erlc_opts: "warnings_as_errors"
+            coverage: true
           - otp_version: "27.1"
             otp_latest: true
             erlc_opts: "warnings_as_errors"
@@ -68,6 +71,11 @@ jobs:
       - name: Elixir test suite
         run: make test_elixir
         continue-on-error: ${{ matrix.development }}
+        env:
+          COVER: "${{ matrix.coverage }}"
+      - name: "Calculate Coverage"
+        run: make cover
+        if: "${{ matrix.coverage }}"
       - name: Build docs (ExDoc main)
         if: ${{ matrix.otp_latest }}
         run: |
@@ -85,6 +93,12 @@ jobs:
           # Recompile System without .git
           cd lib/elixir && ../../bin/elixirc -o ebin lib/system.ex && cd -
           taskset 1 make check_reproducible
+      - name: "Upload Coverage Artifact"
+        if: "${{ matrix.coverage }}"
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: TestCoverage
+          path: cover/*
 
   test_windows:
     name: Windows Server 2019, Erlang/OTP ${{ matrix.otp_version }}

.gitignore

@@ -15,3 +15,4 @@
 /.eunit
 .elixir.plt
 erl_crash.dump
+/cover/

Makefile

@@ -25,6 +25,7 @@ GIT_REVISION = $(strip $(shell git rev-parse HEAD 2> /dev/null ))
 GIT_TAG = $(strip $(shell head="$(call GIT_REVISION)"; git tag --points-at $$head 2> /dev/null | grep -v latest | tail -1))
 SOURCE_DATE_EPOCH_PATH = lib/elixir/tmp/ebin_reproducible
 SOURCE_DATE_EPOCH_FILE = $(SOURCE_DATE_EPOCH_PATH)/SOURCE_DATE_EPOCH
+ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 
 .PHONY: install install_man build_plt clean_plt dialyze test check_reproducible clean clean_elixir clean_man format docs Docs.zip Precompiled.zip zips
 .NOTPARALLEL:
@@ -53,6 +54,11 @@ lib/$(1)/ebin/Elixir.$(2).beam: $(wildcard lib/$(1)/lib/*.ex) $(wildcard lib/$(1
 test_$(1): test_formatted $(1)
 	@ echo "==> $(1) (ex_unit)"
 	$(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/$(TEST_FILES)";
+
+cover/ex_unit_$(1).coverdata:
+	$(Q) mkdir -p "$(ROOT_DIR)/cover"
+	$(Q) COVER="1" $(MAKE) test_$(1)
+cover/combined.coverdata: cover/ex_unit_$(1).coverdata
 endef
 
 define WRITE_SOURCE_DATE_EPOCH
@@ -175,6 +181,7 @@ clean: clean_man
 	rm -rf lib/mix/test/fixtures/git_sparse_repo/
 	rm -rf lib/mix/test/fixtures/archive/ebin/
 	rm -f erl_crash.dump
+	rm -rf cover
 
 clean_elixir:
 	$(Q) rm -f lib/*/ebin/Elixir.*.beam
@@ -287,6 +294,21 @@ test_stdlib: compile
 		cd lib/elixir && ../../bin/elixir --sname primary -r "test/elixir/test_helper.exs" -pr "test/elixir/**/$(TEST_FILES)"; \
 	fi
 
+cover/ex_unit_elixir.coverdata:
+	$(Q) mkdir -p "$(ROOT_DIR)/cover"
+	$(Q) COVER="1" $(MAKE) test_stdlib
+cover/combined.coverdata: cover/ex_unit_elixir.coverdata
+
+cover/combined.coverdata:
+	$(Q) if [ -z "$(GITHUB_STEP_SUMMARY)" ]; then\
+		bin/elixir ./lib/elixir/scripts/cover.exs;\
+	else\
+		bin/elixir ./lib/elixir/scripts/cover.exs | tee "$(GITHUB_STEP_SUMMARY)";\
+	fi
+
+.PHONY: cover
+cover: cover/combined.coverdata
+
 #==> Dialyzer tasks
 
 DIALYZER_OPTS = --no_check_plt --fullpath -Werror_handling -Wunmatched_returns -Wunderspecs

lib/eex/test/test_helper.exs

@@ -5,6 +5,9 @@
 {line_exclude, line_include} =
   if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []}
 
+Code.require_file("../../elixir/scripts/cover_record.exs", __DIR__)
+CoverageRecorder.maybe_record("eex")
+
 ExUnit.start(
   trace: !!System.get_env("TRACE"),
   include: line_include,

lib/elixir/scripts/cover.exs

@@ -0,0 +1,30 @@
+#!bin/elixir
+
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: 2021 The Elixir Team
+
+Code.require_file("cover_record.exs", __DIR__)
+cover_pid = CoverageRecorder.enable_coverage()
+
+coverdata_inputs =
+  CoverageRecorder.cover_dir() |> Path.join("ex_unit_*.coverdata") |> Path.wildcard()
+
+coverdata_output = Path.join(CoverageRecorder.cover_dir(), "combined.coverdata")
+
+for file <- coverdata_inputs do
+  :ok = :cover.import(String.to_charlist(file))
+end
+
+:ok = :cover.export(String.to_charlist(coverdata_output))
+
+{:ok, _} = Application.ensure_all_started(:mix)
+
+# Silence analyse import messages emitted by cover
+{:ok, string_io} = StringIO.open("")
+Process.group_leader(cover_pid, string_io)
+
+:ok =
+  Mix.Tasks.Test.Coverage.generate_cover_results(
+    output: CoverageRecorder.cover_dir(),
+    summary: [threshold: 0]
+  )

lib/elixir/scripts/cover_record.exs

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: 2021 The Elixir Team
+
+defmodule CoverageRecorder do
+  def enabled? do
+    case System.fetch_env("COVER") do
+      {:ok, truthy} when truthy in ~w[1 true yes y] ->
+        true
+
+      _ ->
+        false
+    end
+  end
+
+  def maybe_record(suite_name) do
+    if enabled?() do
+      record(suite_name)
+    end
+  end
+
+  def enable_coverage do
+    _ = :cover.stop()
+    {:ok, pid} = :cover.start()
+
+    cover_compile_ebins()
+
+    pid
+  end
+
+  def cover_dir, do: Path.join(root_dir(), "cover")
+
+  defp root_dir, do: Path.join(__DIR__, "../../..")
+  defp ebins, do: root_dir() |> Path.join("lib/*/ebin") |> Path.wildcard()
+
+  defp record(suite_name) do
+    file = Path.join(cover_dir(), "ex_unit_#{suite_name}.coverdata")
+
+    enable_coverage()
+
+    System.at_exit(fn _status ->
+      :ok = :cover.export(String.to_charlist(file))
+    end)
+  end
+
+  defp cover_compile_ebins do
+    for ebin <- ebins(),
+        result <- :cover.compile_beam_directory(String.to_charlist(ebin)) do
+      case result do
+        {:ok, _module} ->
+          :ok
+
+        {:error, reason} ->
+          raise "Failed to cover compile directory #{ebin} with reason: #{inspect(reason)}"
+      end
+    end
+  end
+end

lib/elixir/test/elixir/exception_test.exs

@@ -888,6 +888,7 @@ defmodule ExceptionTest do
       assert stack == [{BlameModule, :fun, 1, [line: 13]}]
     end
 
+    @tag :require_ast
     test "annotates args and clauses from mfa" do
       import PathHelpers
 

lib/elixir/test/elixir/kernel/dialyzer_test.exs

@@ -8,6 +8,7 @@ defmodule Kernel.DialyzerTest do
   use ExUnit.Case, async: true
 
   @moduletag :dialyzer
+  @moduletag :require_ast
   import PathHelpers
 
   setup_all do

lib/elixir/test/elixir/module/types/integration_test.exs

@@ -458,6 +458,7 @@ defmodule Module.Types.IntegrationTest do
       assert_warnings(files, warnings)
     end
 
+    @tag :require_ast
     test "String.Chars protocol dispatch" do
       files = %{
         "a.ex" => """
@@ -520,6 +521,7 @@ defmodule Module.Types.IntegrationTest do
       assert_warnings(files, warnings, consolidate_protocols: true)
     end
 
+    @tag :require_ast
     test "Enumerable protocol dispatch" do
       files = %{
         "a.ex" => """

lib/elixir/test/elixir/test_helper.exs

@@ -124,9 +124,21 @@ source_exclude =
     []
   end
 
+Code.require_file("../../scripts/cover_record.exs", __DIR__)
+CoverageRecorder.maybe_record("elixir")
+
+cover_exclude =
+  if CoverageRecorder.enabled?() do
+    [:require_ast]
+  else
+    []
+  end
+
 ExUnit.start(
   trace: !!System.get_env("TRACE"),
   assert_receive_timeout: assert_timeout,
-  exclude: epmd_exclude ++ os_exclude ++ line_exclude ++ distributed_exclude ++ source_exclude,
+  exclude:
+    epmd_exclude ++
+      os_exclude ++ line_exclude ++ distributed_exclude ++ source_exclude ++ cover_exclude,
   include: line_include
 )

lib/ex_unit/lib/ex_unit/diff.ex

@@ -1162,6 +1162,7 @@ defmodule ExUnit.Diff do
     else
       other
       |> Map.to_list()
+      |> Enum.sort()
       |> Enum.map(&escape_pair/1)
       |> build_map_or_struct(struct)
     end

lib/ex_unit/test/ex_unit/formatter_test.exs

@@ -86,7 +86,9 @@ defmodule ExUnit.FormatterTest do
 
     failure = [{:exit, {{error, stack}, {:mod, :fun, []}}, []}]
 
-    assert trim_multiline_whitespace(format_test_failure(test(), failure, 1, 80, &formatter/2)) =~
+    format = trim_multiline_whitespace(format_test_failure(test(), failure, 1, 80, &formatter/2))
+
+    assert format =~
              """
                1) world (Hello)
                   test/ex_unit/formatter_test.exs:1
@@ -101,11 +103,16 @@ defmodule ExUnit.FormatterTest do
 
                            # 2
                            :bar
+             """
 
-                       Attempted function clauses (showing 5 out of 5):
+    if Access not in :cover.modules() do
+      assert format =~
+               """
+                         Attempted function clauses (showing 5 out of 5):
 
-                           def fetch(%module{} = container, key)
-             """
+                             def fetch(%module{} = container, key)
+               """
+    end
   end
 
   test "formats test exits with assertion mfa" do
@@ -177,11 +184,16 @@ defmodule ExUnit.FormatterTest do
 
                            # 2
                            :bar
+             """
 
-                       Attempted function clauses (showing 5 out of 5):
+    if Access not in :cover.modules() do
+      assert format =~
+               """
+                         Attempted function clauses (showing 5 out of 5):
 
-                           def fetch(%module{} = container, key)
-             """
+                             def fetch(%module{} = container, key)
+               """
+    end
 
     assert format =~ ~r"lib/access.ex:\d+: Access.fetch/2"
   end
@@ -418,11 +430,16 @@ defmodule ExUnit.FormatterTest do
 
                     # 2
                     :bar
+           """
 
-                Attempted function clauses (showing 5 out of 5):
+    if Access not in :cover.modules() do
+      assert failure =~
+               """
+                    Attempted function clauses (showing 5 out of 5):
 
-                    def fetch(%module{} = container, key)
-           """
+                        def fetch(%module{} = container, key)
+               """
+    end
 
     assert failure =~ ~r"\(elixir #{System.version()}\) lib/access\.ex:\d+: Access\.fetch/2"
   end

lib/ex_unit/test/test_helper.exs

@@ -7,6 +7,9 @@ Logger.configure_backend(:console, colors: [enabled: false])
 {line_exclude, line_include} =
   if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []}
 
+Code.require_file("../../elixir/scripts/cover_record.exs", __DIR__)
+CoverageRecorder.maybe_record("ex_unit")
+
 ExUnit.start(
   trace: !!System.get_env("TRACE"),
   include: line_include,

lib/iex/test/iex/helpers_test.exs

@@ -168,10 +168,12 @@ defmodule IEx.HelpersTest do
                ~r/#{@example_module_path}:\d+$/
     end
 
+    @tag :require_ast
     test "opens function" do
       assert capture_iex("open(h)") |> maybe_trim_quotes() =~ ~r/#{@iex_helpers}:\d+$/
     end
 
+    @tag :require_ast
     test "opens function/arity" do
       assert capture_iex("open(b/1)") |> maybe_trim_quotes() =~ ~r/#{@iex_helpers}:\d+$/
       assert capture_iex("open(h/0)") |> maybe_trim_quotes() =~ ~r/#{@iex_helpers}:\d+$/
@@ -193,14 +195,17 @@ defmodule IEx.HelpersTest do
                ~r/#{@example_module_path}:\d+$/
     end
 
+    @tag :require_ast
     test "opens Erlang module" do
       assert capture_iex("open(:elixir)") |> maybe_trim_quotes() =~ ~r/#{@elixir_erl}:\d+$/
     end
 
+    @tag :require_ast
     test "opens Erlang module.function" do
       assert capture_iex("open(:elixir.start)") |> maybe_trim_quotes() =~ ~r/#{@elixir_erl}:\d+$/
     end
 
+    @tag :require_ast
     test "opens Erlang module.function/arity" do
       assert capture_iex("open(:elixir.start/2)") |> maybe_trim_quotes() =~
                ~r/#{@elixir_erl}:\d+$/

lib/iex/test/test_helper.exs

@@ -32,11 +32,21 @@ source_exclude =
     []
   end
 
+Code.require_file("../../elixir/scripts/cover_record.exs", __DIR__)
+CoverageRecorder.maybe_record("iex")
+
+cover_exclude =
+  if CoverageRecorder.enabled?() do
+    [:require_ast]
+  else
+    []
+  end
+
 ExUnit.start(
   assert_receive_timeout: assert_timeout,
   trace: !!System.get_env("TRACE"),
   include: line_include,
-  exclude: line_exclude ++ erlang_doc_exclude ++ source_exclude
+  exclude: line_exclude ++ erlang_doc_exclude ++ source_exclude ++ cover_exclude
 )
 
 defmodule IEx.Case do