Skip to content

Commit 66ee072

Browse files
committed
Implement Test Coverage Reporting
1 parent 624ae0c commit 66ee072

23 files changed

+481
-199
lines changed

.github/workflows/ci.yml

+15-1
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,16 @@ permissions:
2121

2222
jobs:
2323
test_linux:
24-
name: Ubuntu 24.04, Erlang/OTP ${{ matrix.otp_version }}${{ matrix.deterministic && ' (deterministic)' || '' }}
24+
name: Ubuntu 24.04, Erlang/OTP ${{ matrix.otp_version }}${{ matrix.deterministic && ' (deterministic)' || '' }}${{ matrix.coverage && ' (coverage)' || '' }}
2525
strategy:
2626
fail-fast: false
2727
matrix:
2828
include:
2929
- otp_version: "27.1"
3030
deterministic: true
31+
- otp_version: "27.1"
32+
erlc_opts: "warnings_as_errors"
33+
coverage: true
3134
- otp_version: "27.1"
3235
otp_latest: true
3336
erlc_opts: "warnings_as_errors"
@@ -65,9 +68,14 @@ jobs:
6568
- name: Erlang test suite
6669
run: make test_erlang
6770
continue-on-error: ${{ matrix.development }}
71+
if: "${{ !matrix.coverage }}"
6872
- name: Elixir test suite
6973
run: make test_elixir
7074
continue-on-error: ${{ matrix.development }}
75+
if: "${{ !matrix.coverage }}"
76+
- name: "Calculate Coverage"
77+
run: make cover
78+
if: "${{ matrix.coverage }}"
7179
- name: Build docs (ExDoc main)
7280
if: ${{ matrix.otp_latest }}
7381
run: |
@@ -85,6 +93,12 @@ jobs:
8593
# Recompile System without .git
8694
cd lib/elixir && ../../bin/elixirc -o ebin lib/system.ex && cd -
8795
taskset 1 make check_reproducible
96+
- name: "Upload Coverage Artifact"
97+
if: "${{ matrix.coverage }}"
98+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
99+
with:
100+
name: TestCoverage
101+
path: cover/*
88102

89103
test_windows:
90104
name: Windows Server 2019, Erlang/OTP ${{ matrix.otp_version }}

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,4 @@
1515
/.eunit
1616
.elixir.plt
1717
erl_crash.dump
18+
/cover/

Makefile

+18
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ GIT_REVISION = $(strip $(shell git rev-parse HEAD 2> /dev/null ))
2525
GIT_TAG = $(strip $(shell head="$(call GIT_REVISION)"; git tag --points-at $$head 2> /dev/null | grep -v latest | tail -1))
2626
SOURCE_DATE_EPOCH_PATH = lib/elixir/tmp/ebin_reproducible
2727
SOURCE_DATE_EPOCH_FILE = $(SOURCE_DATE_EPOCH_PATH)/SOURCE_DATE_EPOCH
28+
ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
2829

2930
.PHONY: install install_man build_plt clean_plt dialyze test check_reproducible clean clean_elixir clean_man format docs Docs.zip Precompiled.zip zips
3031
.NOTPARALLEL:
@@ -53,6 +54,11 @@ lib/$(1)/ebin/Elixir.$(2).beam: $(wildcard lib/$(1)/lib/*.ex) $(wildcard lib/$(1
5354
test_$(1): test_formatted $(1)
5455
@ echo "==> $(1) (ex_unit)"
5556
$(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/$(TEST_FILES)";
57+
58+
cover/ex_unit_$(1).coverdata:
59+
$(Q) mkdir -p "$(ROOT_DIR)/cover"
60+
$(Q) COVER_FILE="$(ROOT_DIR)/cover/ex_unit_$(1).coverdata" $(MAKE) test_$(1)
61+
cover/combined.coverdata: cover/ex_unit_$(1).coverdata
5662
endef
5763

5864
define WRITE_SOURCE_DATE_EPOCH
@@ -175,6 +181,7 @@ clean: clean_man
175181
rm -rf lib/mix/test/fixtures/git_sparse_repo/
176182
rm -rf lib/mix/test/fixtures/archive/ebin/
177183
rm -f erl_crash.dump
184+
rm -rf cover
178185

179186
clean_elixir:
180187
$(Q) rm -f lib/*/ebin/Elixir.*.beam
@@ -287,6 +294,17 @@ test_stdlib: compile
287294
cd lib/elixir && ../../bin/elixir --sname primary -r "test/elixir/test_helper.exs" -pr "test/elixir/**/$(TEST_FILES)"; \
288295
fi
289296

297+
cover/ex_unit_stdlib.coverdata:
298+
$(Q) mkdir -p "$(ROOT_DIR)/cover"
299+
$(Q) COVER_FILE="$(ROOT_DIR)/cover/ex_unit_stdlib.coverdata" $(MAKE) test_stdlib
300+
cover/combined.coverdata: cover/ex_unit_stdlib.coverdata
301+
302+
cover/combined.coverdata:
303+
$(Q) bin/elixir ./lib/elixir/scripts/cover.exs
304+
305+
.PHONY: cover
306+
cover: cover/combined.coverdata
307+
290308
#==> Dialyzer tasks
291309

292310
DIALYZER_OPTS = --no_check_plt --fullpath -Werror_handling -Wunmatched_returns -Wunderspecs

lib/eex/test/test_helper.exs

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
{line_exclude, line_include} =
66
if line = System.get_env("LINE"), do: {[:test], [line: line]}, else: {[], []}
77

8+
Code.eval_file("../../elixir/scripts/cover_record.exs", __DIR__)
9+
810
ExUnit.start(
911
trace: !!System.get_env("TRACE"),
1012
include: line_include,

lib/elixir/scripts/cover.exs

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!bin/elixir
2+
3+
# SPDX-License-Identifier: Apache-2.0
4+
# SPDX-FileCopyrightText: 2021 The Elixir Team
5+
6+
root_dir = __ENV__.file |> Path.dirname() |> Path.join("../../..")
7+
cover_dir = Path.join(root_dir, "cover")
8+
coverdata_inputs = cover_dir |> Path.join("ex_unit_*.coverdata") |> Path.wildcard()
9+
coverdata_output = Path.join(cover_dir, "combined.coverdata")
10+
ebins = root_dir |> Path.join("lib/*/ebin") |> Path.wildcard()
11+
12+
_ = :cover.stop()
13+
{:ok, cover_pid} = :cover.start()
14+
15+
for ebin <- ebins,
16+
result <- :cover.compile_beam_directory(String.to_charlist(ebin)) do
17+
case result do
18+
{:ok, _module} ->
19+
:ok
20+
21+
{:error, reason} ->
22+
raise "Failed to cover compile directory #{ebin} with reason: #{inspect(reason)}"
23+
end
24+
end
25+
26+
for file <- coverdata_inputs do
27+
:ok = :cover.import(String.to_charlist(file))
28+
end
29+
30+
:ok = :cover.export(String.to_charlist(coverdata_output))
31+
32+
{:ok, _} = Application.ensure_all_started(:mix)
33+
34+
# Silence analyse import messages emitted by cover
35+
{:ok, string_io} = StringIO.open("")
36+
Process.group_leader(cover_pid, string_io)
37+
38+
:ok =
39+
Mix.Tasks.Test.Coverage.generate_cover_results(
40+
output: cover_dir,
41+
summary: [threshold: 0]
42+
)

lib/elixir/scripts/cover_record.exs

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
# SPDX-FileCopyrightText: 2021 The Elixir Team
3+
4+
root_dir = __ENV__.file |> Path.dirname() |> Path.join("../../..")
5+
ebins = root_dir |> Path.join("lib/*/ebin") |> Path.wildcard()
6+
7+
case System.fetch_env("COVER_FILE") do
8+
{:ok, file} ->
9+
_ = :cover.stop()
10+
{:ok, _pid} = :cover.start()
11+
12+
for ebin <- ebins,
13+
result <- :cover.compile_beam_directory(String.to_charlist(ebin)) do
14+
case result do
15+
{:ok, _module} ->
16+
:ok
17+
18+
{:error, reason} ->
19+
raise "Failed to cover compile directory #{ebin} with reason: #{inspect(reason)}"
20+
end
21+
end
22+
23+
System.at_exit(fn _status ->
24+
:ok = :cover.export(String.to_charlist(file))
25+
end)
26+
27+
true
28+
29+
:error ->
30+
false
31+
end

lib/elixir/scripts/path_helpers.exs

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
# SPDX-FileCopyrightText: 2021 The Elixir Team
3+
# SPDX-FileCopyrightText: 2012 Plataformatec
4+
5+
# Beam files compiled on demand
6+
path = Path.expand("../tmp/beams", __DIR__)
7+
File.rm_rf!(path)
8+
File.mkdir_p!(path)
9+
Code.prepend_path(path)
10+
11+
defmodule PathHelpers do
12+
def fixture_path() do
13+
Path.expand("../test/elixir/fixtures", __DIR__)
14+
end
15+
16+
def tmp_path() do
17+
Path.expand("../tmp", __DIR__)
18+
end
19+
20+
def fixture_path(extra) do
21+
Path.join(fixture_path(), extra)
22+
end
23+
24+
def tmp_path(extra) do
25+
Path.join(tmp_path(), extra)
26+
end
27+
28+
def elixir(args, executable_extension \\ "") do
29+
run_cmd(elixir_executable(executable_extension), args)
30+
end
31+
32+
def elixir_executable(extension \\ "") do
33+
executable_path("elixir", extension)
34+
end
35+
36+
def elixirc(args, executable_extension \\ "") do
37+
run_cmd(elixirc_executable(executable_extension), args)
38+
end
39+
40+
def elixirc_executable(extension \\ "") do
41+
executable_path("elixirc", extension)
42+
end
43+
44+
def iex(args, executable_extension \\ "") do
45+
run_cmd(iex_executable(executable_extension), args)
46+
end
47+
48+
def iex_executable(extension \\ "") do
49+
executable_path("iex", extension)
50+
end
51+
52+
def write_beam({:module, name, bin, _} = res) do
53+
File.mkdir_p!(unquote(path))
54+
beam_path = Path.join(unquote(path), Atom.to_string(name) <> ".beam")
55+
File.write!(beam_path, bin)
56+
res
57+
end
58+
59+
defp run_cmd(executable, args) do
60+
~c"#{executable} #{IO.chardata_to_string(args)}#{redirect_std_err_on_win()}"
61+
|> :os.cmd()
62+
|> :unicode.characters_to_binary()
63+
end
64+
65+
defp executable_path(name, extension) do
66+
Path.expand("../../../bin/#{name}#{extension}", __DIR__)
67+
end
68+
69+
if match?({:win32, _}, :os.type()) do
70+
def windows?, do: true
71+
def executable_extension, do: ".bat"
72+
def redirect_std_err_on_win, do: " 2>&1"
73+
else
74+
def windows?, do: false
75+
def executable_extension, do: ""
76+
def redirect_std_err_on_win, do: ""
77+
end
78+
end

lib/elixir/test/elixir/exception_test.exs

+39-10
Original file line numberDiff line numberDiff line change
@@ -556,20 +556,36 @@ defmodule ExceptionTest do
556556
end
557557

558558
test "annotates function clause errors" do
559-
assert blame_message(Access, & &1.fetch(:foo, :bar)) =~ """
560-
no function clause matching in Access.fetch/2
559+
import PathHelpers
560+
561+
write_beam(
562+
defmodule ExampleModule do
563+
def fun(arg1, arg2)
564+
def fun(:one, :one), do: :ok
565+
def fun(:two, :two), do: :ok
566+
end
567+
)
568+
569+
:code.purge(ExampleModule)
570+
:code.delete(ExampleModule)
571+
572+
message = blame_message(ExceptionTest.ExampleModule, & &1.fun(:three, :four))
573+
574+
assert message =~ """
575+
no function clause matching in ExceptionTest.ExampleModule.fun/2
561576
562-
The following arguments were given to Access.fetch/2:
577+
The following arguments were given to ExceptionTest.ExampleModule.fun/2:
563578
564579
# 1
565-
:foo
580+
:three
566581
567582
# 2
568-
:bar
583+
:four
569584
570-
Attempted function clauses (showing 5 out of 5):
585+
Attempted function clauses (showing 2 out of 2):
571586
572-
def fetch(-%module{} = container-, key)
587+
def fun(-:one-, -:one-)
588+
def fun(-:two-, -:two-)
573589
"""
574590
end
575591

@@ -858,15 +874,28 @@ defmodule ExceptionTest do
858874

859875
describe "blaming unit tests" do
860876
test "annotates clauses errors" do
861-
args = [%{}, :key, nil]
877+
import PathHelpers
878+
879+
write_beam(
880+
defmodule ExampleModule do
881+
def fun(arg), do: arg
882+
end
883+
)
884+
885+
:code.purge(ExampleModule)
886+
:code.delete(ExampleModule)
887+
888+
args = [nil]
862889

863890
{exception, stack} =
864-
Exception.blame(:error, :function_clause, [{Keyword, :pop, args, [line: 13]}])
891+
Exception.blame(:error, :function_clause, [{ExampleModule, :fun, args, [line: 13]}])
865892

866893
assert %FunctionClauseError{kind: :def, args: ^args, clauses: [_]} = exception
867-
assert stack == [{Keyword, :pop, 3, [line: 13]}]
894+
895+
assert stack == [{ExampleModule, :fun, 1, [line: 13]}]
868896
end
869897

898+
@tag :require_ast
870899
test "annotates args and clauses from mfa" do
871900
import PathHelpers
872901

lib/elixir/test/elixir/kernel/dialyzer_test.exs

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ defmodule Kernel.DialyzerTest do
88
use ExUnit.Case, async: true
99

1010
@moduletag :dialyzer
11+
@moduletag :require_ast
1112
import PathHelpers
1213

1314
setup_all do

lib/elixir/test/elixir/module/types/integration_test.exs

+2
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,7 @@ defmodule Module.Types.IntegrationTest do
458458
assert_warnings(files, warnings)
459459
end
460460

461+
@tag :require_ast
461462
test "String.Chars protocol dispatch" do
462463
files = %{
463464
"a.ex" => """
@@ -520,6 +521,7 @@ defmodule Module.Types.IntegrationTest do
520521
assert_warnings(files, warnings, consolidate_protocols: true)
521522
end
522523

524+
@tag :require_ast
523525
test "Enumerable protocol dispatch" do
524526
files = %{
525527
"a.ex" => """

lib/elixir/test/elixir/module_test.exs

+14-1
Original file line numberDiff line numberDiff line change
@@ -431,8 +431,21 @@ defmodule ModuleTest do
431431
assert backend.debug_info(:elixir_v1, ModuleCreateNoDebugInfo, data, []) == {:error, :missing}
432432
end
433433

434+
@tag :require_ast
434435
test "compiles to core" do
435-
{:ok, {Atom, [{~c"Dbgi", dbgi}]}} = Atom |> :code.which() |> :beam_lib.chunks([~c"Dbgi"])
436+
import PathHelpers
437+
438+
write_beam(
439+
defmodule ExampleModule do
440+
end
441+
)
442+
443+
:code.purge(ExampleModule)
444+
:code.delete(ExampleModule)
445+
446+
{:ok, {Atom, [{~c"Dbgi", dbgi}]}} =
447+
ExampleModule |> :code.which() |> :beam_lib.chunks([~c"Dbgi"])
448+
436449
{:debug_info_v1, backend, data} = :erlang.binary_to_term(dbgi)
437450
{:ok, core} = backend.debug_info(:core_v1, Atom, data, [])
438451
assert is_tuple(core)

0 commit comments

Comments
 (0)