Merge pull request #318 from dnephin/slowest-n-tests

slowest: Add num flag for limiting the number of slow tests to print

Author: dnephin

README.md

@@ -172,6 +172,20 @@ quoting the whole command.
 gotestsum --post-run-command "notify me --date"
 ```
 
+**Example: printing slowest tests**
+
+The post-run command can be combined with other `gotestsum` commands and tools to provide
+a more detailed summary. This example uses `gotestsum tool slowest` to print the
+slowest 10 tests after the summary.
+
+```
+gotestsum \
+  --jsonfile tmp.json.log \
+  --post-run-command "bash -c '
+    echo; echo Slowest tests;
+    gotestsum tool slowest --num 10 --jsonfile tmp.json.log'"
+```
+
 ### Re-running failed tests
 
 When the `--rerun-fails` flag is set, `gotestsum` will re-run any failed tests.

cmd/tool/slowest/slowest.go

@@ -37,6 +37,8 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
 		"path to test2json output, defaults to stdin")
 	flags.DurationVar(&opts.threshold, "threshold", 100*time.Millisecond,
 		"test cases with elapsed time greater than threshold are slow tests")
+	flags.IntVar(&opts.topN, "num", 0,
+		"print at most num slowest tests, instead of all tests above the threshold")
 	flags.StringVar(&opts.skipStatement, "skip-stmt", "",
 		"add this go statement to slow tests, instead of printing the list of slow tests")
 	flags.BoolVar(&opts.debug, "debug", false,
@@ -94,6 +96,7 @@ Flags:
 
 type options struct {
 	threshold     time.Duration
+	topN          int
 	jsonfile      string
 	skipStatement string
 	debug         bool
@@ -118,7 +121,7 @@ func run(opts *options) error {
 		return fmt.Errorf("failed to scan testjson: %v", err)
 	}
 
-	tcs := aggregate.Slowest(exec, opts.threshold)
+	tcs := aggregate.Slowest(exec, opts.threshold, opts.topN)
 	if opts.skipStatement != "" {
 		skipStmt, err := parseSkipStatement(opts.skipStatement)
 		if err != nil {

cmd/tool/slowest/testdata/cmd-flags-help-text

@@ -42,5 +42,6 @@ https://golang.org/cmd/go/#hdr-Environment_variables.
 Flags:
       --debug                enable debug logging.
       --jsonfile string      path to test2json output, defaults to stdin
+      --num int              print at most num slowest tests, instead of all tests above the threshold
       --skip-stmt string     add this go statement to slow tests, instead of printing the list of slow tests
       --threshold duration   test cases with elapsed time greater than threshold are slow tests (default 100ms)

internal/aggregate/slowest.go

@@ -13,8 +13,8 @@ import (
 //
 // If there are multiple runs of a TestCase, all of them will be represented
 // by a single TestCase with the median elapsed time in the returned slice.
-func Slowest(exec *testjson.Execution, threshold time.Duration) []testjson.TestCase {
-	if threshold == 0 {
+func Slowest(exec *testjson.Execution, threshold time.Duration, num int) []testjson.TestCase {
+	if threshold == 0 && num == 0 {
 		return nil
 	}
 	pkgs := exec.Packages()
@@ -26,6 +26,13 @@ func Slowest(exec *testjson.Execution, threshold time.Duration) []testjson.TestC
 	sort.Slice(tests, func(i, j int) bool {
 		return tests[i].Elapsed > tests[j].Elapsed
 	})
+	if num >= len(tests) {
+		return tests
+	}
+	if num > 0 {
+		return tests[:num]
+	}
+
 	end := sort.Search(len(tests), func(i int) bool {
 		return tests[i].Elapsed < threshold
 	})

internal/aggregate/slowest_test.go

@@ -1,15 +1,115 @@
 package aggregate
 
 import (
+	"bytes"
+	"encoding/json"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"gotest.tools/gotestsum/testjson"
 	"gotest.tools/v3/assert"
 )
 
+func TestSlowest(t *testing.T) {
+	newEvent := func(pkg, test string, elapsed float64) testjson.TestEvent {
+		return testjson.TestEvent{
+			Package: pkg,
+			Test:    test,
+			Action:  testjson.ActionPass,
+			Elapsed: elapsed,
+		}
+	}
+
+	exec := newExecutionFromEvents(t,
+		newEvent("one", "TestOmega", 22.2),
+		newEvent("one", "TestOmega", 1.5),
+		newEvent("one", "TestOmega", 0.6),
+		newEvent("one", "TestOnion", 0.5),
+		newEvent("two", "TestTents", 2.5),
+		newEvent("two", "TestTin", 0.3),
+		newEvent("two", "TestTunnel", 1.1))
+
+	cmpCasesShallow := cmp.Comparer(func(x, y testjson.TestCase) bool {
+		return x.Package == y.Package && x.Test == y.Test
+	})
+
+	type testCase struct {
+		name      string
+		threshold time.Duration
+		num       int
+		expected  []testjson.TestCase
+	}
+
+	run := func(t *testing.T, tc testCase) {
+		actual := Slowest(exec, tc.threshold, tc.num)
+		assert.DeepEqual(t, actual, tc.expected, cmpCasesShallow)
+	}
+
+	testCases := []testCase{
+		{
+			name:      "threshold only",
+			threshold: time.Second,
+			expected: []testjson.TestCase{
+				{Package: "two", Test: "TestTents"},
+				{Package: "one", Test: "TestOmega"},
+				{Package: "two", Test: "TestTunnel"},
+			},
+		},
+		{
+			name:      "threshold only 2s",
+			threshold: 2 * time.Second,
+			expected: []testjson.TestCase{
+				{Package: "two", Test: "TestTents"},
+			},
+		},
+		{
+			name:      "threshold and num",
+			threshold: 400 * time.Millisecond,
+			num:       2,
+			expected: []testjson.TestCase{
+				{Package: "two", Test: "TestTents"},
+				{Package: "one", Test: "TestOmega"},
+			},
+		},
+		{
+			name: "num only",
+			num:  4,
+			expected: []testjson.TestCase{
+				{Package: "two", Test: "TestTents"},
+				{Package: "one", Test: "TestOmega"},
+				{Package: "two", Test: "TestTunnel"},
+				{Package: "one", Test: "TestOnion"},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			run(t, tc)
+		})
+	}
+}
+
+func newExecutionFromEvents(t *testing.T, events ...testjson.TestEvent) *testjson.Execution {
+	t.Helper()
+
+	buf := new(bytes.Buffer)
+	encoder := json.NewEncoder(buf)
+	for i, event := range events {
+		assert.NilError(t, encoder.Encode(event), "event %d", i)
+	}
+
+	exec, err := testjson.ScanTestOutput(testjson.ScanConfig{
+		Stdout: buf,
+		Stderr: strings.NewReader(""),
+	})
+	assert.NilError(t, err)
+	return exec
+}
+
 func TestByElapsed_WithMedian(t *testing.T) {
 	cases := []testjson.TestCase{
 		{Test: "TestOne", Package: "pkg", Elapsed: time.Second},