From b3cdd389e3b6bf9a69893e9c3487d176530e314f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20M=C3=A4nnchen?= Date: Tue, 18 Mar 2025 14:49:52 +0000 Subject: [PATCH] Implement Test Coverage Reporting --- .github/workflows/ci.yml | 16 +++- .gitignore | 1 + Makefile | 15 ++++ lib/eex/test/test_helper.exs | 3 + lib/elixir/scripts/cover.exs | 30 ++++++++ lib/elixir/scripts/cover_record.exs | 77 +++++++++++++++++++ lib/elixir/test/elixir/exception_test.exs | 1 + .../test/elixir/kernel/dialyzer_test.exs | 1 + .../elixir/module/types/integration_test.exs | 2 + lib/elixir/test/elixir/test_helper.exs | 13 +++- lib/ex_unit/lib/ex_unit/diff.ex | 1 + lib/ex_unit/test/ex_unit/formatter_test.exs | 37 ++++++--- lib/ex_unit/test/test_helper.exs | 3 + lib/iex/test/iex/helpers_test.exs | 5 ++ lib/iex/test/test_helper.exs | 11 ++- lib/logger/test/test_helper.exs | 3 + lib/mix/lib/mix/tasks/test.coverage.ex | 3 +- lib/mix/test/test_helper.exs | 3 + 18 files changed, 211 insertions(+), 14 deletions(-) create mode 100755 lib/elixir/scripts/cover.exs create mode 100644 lib/elixir/scripts/cover_record.exs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be10cf55ded..f8239cf5f50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,13 +21,16 @@ permissions: jobs: test_linux: - name: Ubuntu 24.04, Erlang/OTP ${{ matrix.otp_version }}${{ matrix.deterministic && ' (deterministic)' || '' }} + name: Ubuntu 24.04, Erlang/OTP ${{ matrix.otp_version }}${{ matrix.deterministic && ' (deterministic)' || '' }}${{ matrix.coverage && ' (coverage)' || '' }} strategy: fail-fast: false matrix: include: - otp_version: "27.1" deterministic: true + - otp_version: "27.1" + erlc_opts: "warnings_as_errors" + coverage: true - otp_version: "27.1" otp_latest: true erlc_opts: "warnings_as_errors" @@ -68,6 +71,11 @@ jobs: - name: Elixir test suite run: make test_elixir continue-on-error: ${{ matrix.development }} + env: + COVER: "${{ matrix.coverage }}" + - name: "Calculate Coverage" + run: make cover | tee "$GITHUB_STEP_SUMMARY" + if: "${{ matrix.coverage }}" - name: Build docs (ExDoc main) if: ${{ matrix.otp_latest }} run: | @@ -85,6 +93,12 @@ jobs: # Recompile System without .git cd lib/elixir && ../../bin/elixirc -o ebin lib/system.ex && cd - taskset 1 make check_reproducible + - name: "Upload Coverage Artifact" + if: "${{ matrix.coverage }}" + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: TestCoverage + path: cover/* test_windows: name: Windows Server 2019, Erlang/OTP ${{ matrix.otp_version }} diff --git a/.gitignore b/.gitignore index 4b40c1790b5..5173250d4d4 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ /.eunit .elixir.plt erl_crash.dump +/cover/ diff --git a/Makefile b/Makefile index f1829b96046..855a7ed4f09 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,10 @@ lib/$(1)/ebin/Elixir.$(2).beam: $(wildcard lib/$(1)/lib/*.ex) $(wildcard lib/$(1 test_$(1): test_formatted $(1) @ echo "==> $(1) (ex_unit)" $(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/$(TEST_FILES)"; + +cover/ex_unit_$(1).coverdata: + $(Q) COVER="1" $(MAKE) test_$(1) +cover/combined.coverdata: cover/ex_unit_$(1).coverdata endef define WRITE_SOURCE_DATE_EPOCH @@ -175,6 +179,7 @@ clean: clean_man rm -rf lib/mix/test/fixtures/git_sparse_repo/ rm -rf lib/mix/test/fixtures/archive/ebin/ rm -f erl_crash.dump + rm -rf cover clean_elixir: $(Q) rm -f lib/*/ebin/Elixir.*.beam @@ -287,6 +292,16 @@ test_stdlib: compile cd lib/elixir && ../../bin/elixir --sname primary -r "test/elixir/test_helper.exs" -pr "test/elixir/**/$(TEST_FILES)"; \ fi +cover/ex_unit_elixir.coverdata: + $(Q) COVER="1" $(MAKE) test_stdlib +cover/combined.coverdata: cover/ex_unit_elixir.coverdata + +cover/combined.coverdata: + bin/elixir ./lib/elixir/scripts/cover.exs + +.PHONY: cover +cover: cover/combined.coverdata + #==> Dialyzer tasks DIALYZER_OPTS = --no_check_plt --fullpath -Werror_handling -Wunmatched_returns -Wunderspecs diff --git a/lib/eex/test/test_helper.exs b/lib/eex/test/test_helper.exs index 4956567b9e1..4d157609935 100644 --- a/lib/eex/test/test_helper.exs +++ b/lib/eex/test/test_helper.exs @@ -5,6 +5,9 @@ {line_exclude, line_include} = if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []} +Code.require_file("../../elixir/scripts/cover_record.exs", __DIR__) +CoverageRecorder.maybe_record("eex") + ExUnit.start( trace: !!System.get_env("TRACE"), include: line_include, diff --git a/lib/elixir/scripts/cover.exs b/lib/elixir/scripts/cover.exs new file mode 100755 index 00000000000..a00973dd8f8 --- /dev/null +++ b/lib/elixir/scripts/cover.exs @@ -0,0 +1,30 @@ +#!bin/elixir + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +Code.require_file("cover_record.exs", __DIR__) +cover_pid = CoverageRecorder.enable_coverage() + +coverdata_inputs = + CoverageRecorder.cover_dir() |> Path.join("ex_unit_*.coverdata") |> Path.wildcard() + +coverdata_output = Path.join(CoverageRecorder.cover_dir(), "combined.coverdata") + +for file <- coverdata_inputs do + :ok = :cover.import(String.to_charlist(file)) +end + +:ok = :cover.export(String.to_charlist(coverdata_output)) + +{:ok, _} = Application.ensure_all_started(:mix) + +# Silence analyse import messages emitted by cover +{:ok, string_io} = StringIO.open("") +Process.group_leader(cover_pid, string_io) + +:ok = + Mix.Tasks.Test.Coverage.generate_cover_results( + output: CoverageRecorder.cover_dir(), + summary: [threshold: 0] + ) diff --git a/lib/elixir/scripts/cover_record.exs b/lib/elixir/scripts/cover_record.exs new file mode 100644 index 00000000000..234a99e0b77 --- /dev/null +++ b/lib/elixir/scripts/cover_record.exs @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: 2021 The Elixir Team + +defmodule CoverageRecorder do + def maybe_record(suite_name) do + if enabled?() do + record(suite_name) + + true + else + false + end + end + + def enable_coverage do + _ = :cover.stop() + {:ok, pid} = :cover.start() + + cover_compile_ebins() + + pid + end + + def cover_dir, do: Path.join(root_dir(), "cover") + + defp enabled? do + case System.fetch_env("COVER") do + {:ok, truthy} when truthy in ~w[1 true yes y] -> + true + + _ -> + false + end + end + + defp root_dir, do: Path.join(__DIR__, "../../..") + defp ebins, do: root_dir() |> Path.join("lib/*/ebin") |> Path.wildcard() + + defp record(suite_name) do + file = Path.join(cover_dir(), "ex_unit_#{suite_name}.coverdata") + + enable_coverage() + + System.at_exit(fn _status -> + File.mkdir_p!(cover_dir()) + + :ok = :cover.export(String.to_charlist(file)) + end) + end + + defp cover_compile_ebins do + relevant_beam_files() + |> Enum.map(&String.to_charlist/1) + |> :cover.compile_beam() + |> Enum.each(fn + {:ok, _module} -> + :ok + + {:error, reason} -> + raise "Failed to cover compile with reason: #{inspect(reason)}" + end) + end + + defp relevant_beam_files do + ebins() + |> Enum.flat_map(fn ebin -> + ebin |> Path.join("*.beam") |> Path.wildcard() + end) + |> Enum.reject(&deprecated/1) + end + + defp deprecated(file) do + mod = file |> Path.basename(".beam") |> String.to_atom() + + match?({:docs_v1, _, _, _, _, %{deprecated: _}, _}, Code.fetch_docs(mod)) + end +end diff --git a/lib/elixir/test/elixir/exception_test.exs b/lib/elixir/test/elixir/exception_test.exs index e3fd9feb134..6cda607a62a 100644 --- a/lib/elixir/test/elixir/exception_test.exs +++ b/lib/elixir/test/elixir/exception_test.exs @@ -888,6 +888,7 @@ defmodule ExceptionTest do assert stack == [{BlameModule, :fun, 1, [line: 13]}] end + @tag :require_ast test "annotates args and clauses from mfa" do import PathHelpers diff --git a/lib/elixir/test/elixir/kernel/dialyzer_test.exs b/lib/elixir/test/elixir/kernel/dialyzer_test.exs index d414b6d8c66..ba5cbfb5bce 100644 --- a/lib/elixir/test/elixir/kernel/dialyzer_test.exs +++ b/lib/elixir/test/elixir/kernel/dialyzer_test.exs @@ -8,6 +8,7 @@ defmodule Kernel.DialyzerTest do use ExUnit.Case, async: true @moduletag :dialyzer + @moduletag :require_ast import PathHelpers setup_all do diff --git a/lib/elixir/test/elixir/module/types/integration_test.exs b/lib/elixir/test/elixir/module/types/integration_test.exs index 8e67051f251..ff39d5cd9ff 100644 --- a/lib/elixir/test/elixir/module/types/integration_test.exs +++ b/lib/elixir/test/elixir/module/types/integration_test.exs @@ -458,6 +458,7 @@ defmodule Module.Types.IntegrationTest do assert_warnings(files, warnings) end + @tag :require_ast test "String.Chars protocol dispatch" do files = %{ "a.ex" => """ @@ -520,6 +521,7 @@ defmodule Module.Types.IntegrationTest do assert_warnings(files, warnings, consolidate_protocols: true) end + @tag :require_ast test "Enumerable protocol dispatch" do files = %{ "a.ex" => """ diff --git a/lib/elixir/test/elixir/test_helper.exs b/lib/elixir/test/elixir/test_helper.exs index 9ac2de26b87..0b233cf8dec 100644 --- a/lib/elixir/test/elixir/test_helper.exs +++ b/lib/elixir/test/elixir/test_helper.exs @@ -124,9 +124,20 @@ source_exclude = [] end +Code.require_file("../../scripts/cover_record.exs", __DIR__) + +cover_exclude = + if CoverageRecorder.maybe_record("elixir") do + [:require_ast] + else + [] + end + ExUnit.start( trace: !!System.get_env("TRACE"), assert_receive_timeout: assert_timeout, - exclude: epmd_exclude ++ os_exclude ++ line_exclude ++ distributed_exclude ++ source_exclude, + exclude: + epmd_exclude ++ + os_exclude ++ line_exclude ++ distributed_exclude ++ source_exclude ++ cover_exclude, include: line_include ) diff --git a/lib/ex_unit/lib/ex_unit/diff.ex b/lib/ex_unit/lib/ex_unit/diff.ex index b562b646815..b2c9604f16a 100644 --- a/lib/ex_unit/lib/ex_unit/diff.ex +++ b/lib/ex_unit/lib/ex_unit/diff.ex @@ -1162,6 +1162,7 @@ defmodule ExUnit.Diff do else other |> Map.to_list() + |> Enum.sort() |> Enum.map(&escape_pair/1) |> build_map_or_struct(struct) end diff --git a/lib/ex_unit/test/ex_unit/formatter_test.exs b/lib/ex_unit/test/ex_unit/formatter_test.exs index 62ecfc0d2a4..923af02c7f5 100644 --- a/lib/ex_unit/test/ex_unit/formatter_test.exs +++ b/lib/ex_unit/test/ex_unit/formatter_test.exs @@ -86,7 +86,9 @@ defmodule ExUnit.FormatterTest do failure = [{:exit, {{error, stack}, {:mod, :fun, []}}, []}] - assert trim_multiline_whitespace(format_test_failure(test(), failure, 1, 80, &formatter/2)) =~ + format = trim_multiline_whitespace(format_test_failure(test(), failure, 1, 80, &formatter/2)) + + assert format =~ """ 1) world (Hello) test/ex_unit/formatter_test.exs:1 @@ -101,11 +103,16 @@ defmodule ExUnit.FormatterTest do # 2 :bar + """ - Attempted function clauses (showing 5 out of 5): + if Access not in :cover.modules() do + assert format =~ + """ + Attempted function clauses (showing 5 out of 5): - def fetch(%module{} = container, key) - """ + def fetch(%module{} = container, key) + """ + end end test "formats test exits with assertion mfa" do @@ -177,11 +184,16 @@ defmodule ExUnit.FormatterTest do # 2 :bar + """ - Attempted function clauses (showing 5 out of 5): + if Access not in :cover.modules() do + assert format =~ + """ + Attempted function clauses (showing 5 out of 5): - def fetch(%module{} = container, key) - """ + def fetch(%module{} = container, key) + """ + end assert format =~ ~r"lib/access.ex:\d+: Access.fetch/2" end @@ -418,11 +430,16 @@ defmodule ExUnit.FormatterTest do # 2 :bar + """ - Attempted function clauses (showing 5 out of 5): + if Access not in :cover.modules() do + assert failure =~ + """ + Attempted function clauses (showing 5 out of 5): - def fetch(%module{} = container, key) - """ + def fetch(%module{} = container, key) + """ + end assert failure =~ ~r"\(elixir #{System.version()}\) lib/access\.ex:\d+: Access\.fetch/2" end diff --git a/lib/ex_unit/test/test_helper.exs b/lib/ex_unit/test/test_helper.exs index bacef541491..57db58b5898 100644 --- a/lib/ex_unit/test/test_helper.exs +++ b/lib/ex_unit/test/test_helper.exs @@ -7,6 +7,9 @@ Logger.configure_backend(:console, colors: [enabled: false]) {line_exclude, line_include} = if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []} +Code.require_file("../../elixir/scripts/cover_record.exs", __DIR__) +CoverageRecorder.maybe_record("ex_unit") + ExUnit.start( trace: !!System.get_env("TRACE"), include: line_include, diff --git a/lib/iex/test/iex/helpers_test.exs b/lib/iex/test/iex/helpers_test.exs index 11a76f13fcc..594e32cceeb 100644 --- a/lib/iex/test/iex/helpers_test.exs +++ b/lib/iex/test/iex/helpers_test.exs @@ -168,10 +168,12 @@ defmodule IEx.HelpersTest do ~r/#{@example_module_path}:\d+$/ end + @tag :require_ast test "opens function" do assert capture_iex("open(h)") |> maybe_trim_quotes() =~ ~r/#{@iex_helpers}:\d+$/ end + @tag :require_ast test "opens function/arity" do assert capture_iex("open(b/1)") |> maybe_trim_quotes() =~ ~r/#{@iex_helpers}:\d+$/ assert capture_iex("open(h/0)") |> maybe_trim_quotes() =~ ~r/#{@iex_helpers}:\d+$/ @@ -193,14 +195,17 @@ defmodule IEx.HelpersTest do ~r/#{@example_module_path}:\d+$/ end + @tag :require_ast test "opens Erlang module" do assert capture_iex("open(:elixir)") |> maybe_trim_quotes() =~ ~r/#{@elixir_erl}:\d+$/ end + @tag :require_ast test "opens Erlang module.function" do assert capture_iex("open(:elixir.start)") |> maybe_trim_quotes() =~ ~r/#{@elixir_erl}:\d+$/ end + @tag :require_ast test "opens Erlang module.function/arity" do assert capture_iex("open(:elixir.start/2)") |> maybe_trim_quotes() =~ ~r/#{@elixir_erl}:\d+$/ diff --git a/lib/iex/test/test_helper.exs b/lib/iex/test/test_helper.exs index 7b016f140a8..0e4989bdb5d 100644 --- a/lib/iex/test/test_helper.exs +++ b/lib/iex/test/test_helper.exs @@ -32,11 +32,20 @@ source_exclude = [] end +Code.require_file("../../elixir/scripts/cover_record.exs", __DIR__) + +cover_exclude = + if CoverageRecorder.maybe_record("iex") do + [:require_ast] + else + [] + end + ExUnit.start( assert_receive_timeout: assert_timeout, trace: !!System.get_env("TRACE"), include: line_include, - exclude: line_exclude ++ erlang_doc_exclude ++ source_exclude + exclude: line_exclude ++ erlang_doc_exclude ++ source_exclude ++ cover_exclude ) defmodule IEx.Case do diff --git a/lib/logger/test/test_helper.exs b/lib/logger/test/test_helper.exs index 909d28dda11..5ef0f215ffb 100644 --- a/lib/logger/test/test_helper.exs +++ b/lib/logger/test/test_helper.exs @@ -5,6 +5,9 @@ {line_exclude, line_include} = if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []} +Code.require_file("../../elixir/scripts/cover_record.exs", __DIR__) +CoverageRecorder.maybe_record("logger") + ExUnit.start( trace: !!System.get_env("TRACE"), include: line_include, diff --git a/lib/mix/lib/mix/tasks/test.coverage.ex b/lib/mix/lib/mix/tasks/test.coverage.ex index 143a6bc0dff..c85c89adda2 100644 --- a/lib/mix/lib/mix/tasks/test.coverage.ex +++ b/lib/mix/lib/mix/tasks/test.coverage.ex @@ -268,7 +268,8 @@ defmodule Mix.Tasks.Test.Coverage do end end - defp generate_cover_results(opts) do + @doc false + def generate_cover_results(opts) do {:result, ok, _fail} = :cover.analyse(:coverage, :line) ignore = opts[:ignore_modules] || [] modules = Enum.reject(:cover.modules(), &ignored?(&1, ignore)) diff --git a/lib/mix/test/test_helper.exs b/lib/mix/test/test_helper.exs index 05f21bc2df9..fcf9e80f771 100644 --- a/lib/mix/test/test_helper.exs +++ b/lib/mix/test/test_helper.exs @@ -43,6 +43,9 @@ cover_exclude = [] end +Code.require_file("../../elixir/scripts/cover_record.exs", __DIR__) +CoverageRecorder.maybe_record("mix") + ExUnit.start( trace: !!System.get_env("TRACE"), exclude: epmd_exclude ++ os_exclude ++ git_exclude ++ line_exclude ++ cover_exclude,