feat(docker): provide Dockerfile and xUnit output parser

Dockerfile

@@ -0,0 +1,19 @@
+FROM docker.io/library/golang:1.18-alpine3.15 AS reqs
+WORKDIR /app
+RUN apk add --no-cache build-base
+ADD go.mod .
+ADD go.sum .
+ADD hiveproxy .
+RUN go mod download
+
+FROM reqs as builder
+WORKDIR /app
+ADD . .
+RUN go build hive.go
+RUN go build -o hivecioutput cmd/hivecioutput/main.go
+
+FROM builder as runner
+WORKDIR /app
+COPY --from=builder /app/hive .
+COPY --from=builder /app/hivecioutput .
+ENTRYPOINT ["/app/hive"]

cmd/hivecioutput/example-exclusions.json

@@ -0,0 +1,10 @@
+{
+    "testSuites": [
+      {
+        "name": "engine-transition",
+        "testCases": [
+          "Terminal blocks are gossiped"
+        ]
+      }
+    ]
+  }

cmd/hivecioutput/main.go

@@ -0,0 +1,332 @@
+package main
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/ethereum/hive/internal/libhive"
+	"gopkg.in/inconshreveable/log15.v2"
+)
+
+const (
+	Header = `<?xml version="1.0" encoding="UTF-8"?>` + "\n"
+)
+
+type Testsuites struct {
+	XMLName   xml.Name `xml:"testsuites"`
+	Text      string   `xml:",chardata"`
+	ID        string   `xml:"id,attr"`
+	Name      string   `xml:"name,attr"`
+	Tests     int      `xml:"tests,attr"`
+	Failures  int      `xml:"failures,attr"`
+	Skips     int      `xml:"skips,attr"`
+	Time      string   `xml:"time,attr"`
+	Testsuite []Testsuite
+}
+
+type Testsuite struct {
+	XMLName  xml.Name `xml:"testsuite"`
+	Text     string   `xml:",chardata"`
+	ID       int      `xml:"id,attr"`
+	Name     string   `xml:"name,attr"`
+	Tests    int      `xml:"tests,attr"`
+	Failures int      `xml:"failures,attr"`
+	Skips    int      `xml:"skips,attr"`
+	Time     string   `xml:"time,attr"`
+	Testcase []Testcase
+}
+
+type Testcase struct {
+	XMLName xml.Name `xml:"testcase"`
+	Text    string   `xml:",chardata"`
+	ID      int      `xml:"id,attr"`
+	Name    string   `xml:"name,attr"`
+	Time    string   `xml:"time,attr"`
+	Failure *Failure
+	Skipped *Skipped
+}
+
+type Failure struct {
+	XMLName xml.Name `xml:"failure,omitempty"`
+	Text    string   `xml:",chardata"`
+	Message string   `xml:"message,attr"`
+	Type    string   `xml:"type,attr"`
+}
+
+type Skipped struct {
+	XMLName xml.Name `xml:"skipped"`
+	Text    string   `xml:",chardata"`
+	Message string   `xml:"message,attr"`
+	Type    string   `xml:"type,attr"`
+}
+
+type Exclusions struct {
+	TestSuites []struct {
+		Name      string   `json:"name"`
+		TestCases []string `json:"testcases"`
+	} `json:"testSuites"`
+}
+
+func main() {
+	var exitcode bool
+	flag.BoolVar(&exitcode, "exitcode", false, "Return exit code 1 on failed tests")
+
+	var (
+		resultsdir     = flag.String("resultsdir", "/tmp/TestResults", "Results dir to scan")
+		outdir         = flag.String("outdir", "/tmp/", "Output dir for xunit xml")
+		exclusionsFile = flag.String("exclusionsfile", "", "File containing list of test names to exclude")
+	)
+	flag.Parse()
+
+	log15.Info(fmt.Sprintf("loading results from %s", *resultsdir))
+	log15.Info(fmt.Sprintf("outputting xunit xml to %s", *outdir))
+	log15.Info(fmt.Sprintf("return exit code 1 on failed tests %v", exitcode))
+	log15.Info(fmt.Sprintf("exclusions file %s", *exclusionsFile))
+
+	// load exclusions
+	exclusions := &Exclusions{}
+	if *exclusionsFile != "" {
+		file, err := os.Open(*exclusionsFile)
+		if err != nil {
+			log15.Info(fmt.Sprintf("error opening exclusions file %s", *exclusionsFile))
+		}
+		bytes, err := io.ReadAll(file)
+		if err != nil {
+			log15.Info(fmt.Sprintf("error reading exclusions file %s", *exclusionsFile))
+		}
+		err = json.Unmarshal(bytes, &exclusions)
+		if err != nil {
+			log15.Info(fmt.Sprintf("error unmarshalling exclusions file %s", *exclusionsFile))
+		}
+		file.Close()
+
+		// print exclusions
+		if len(exclusions.TestSuites) > 0 {
+			log15.Info(fmt.Sprintf("excluded tests: %v", exclusions))
+		}
+	}
+
+	outputs := []*libhive.TestSuite{}
+
+	rd := os.DirFS(*resultsdir)
+	err := walkSummaryFiles(rd, ".", collectOutput, &outputs)
+	if err != nil {
+		log15.Info(fmt.Errorf("error reading results: %w", err).Error())
+		os.Exit(1)
+	}
+
+	pass, run := outputXUnitXmlFile(&outputs, *outdir, exclusions)
+
+	// tests run and passed
+	if pass && run {
+		log15.Info("tests passed!")
+		os.Exit(0)
+	}
+
+	// no tests run
+	if !run {
+		log15.Info("no tests run!")
+		if exitcode {
+			os.Exit(1)
+		}
+	}
+
+	// tests run but failed
+	log15.Info("tests failed!")
+	if exitcode {
+		os.Exit(1)
+	}
+}
+
+func outputXUnitXmlFile(outputs *[]*libhive.TestSuite, path string, exclusions *Exclusions) (bool, bool) {
+	opTs := Testsuites{}
+	totalTests := 0
+	totalFailures := 0
+	totalSkipped := 0
+	suiteNo := 0
+
+	for _, ts := range *outputs {
+		log15.Info("test suite", "name", ts.Name)
+		tests := 0
+		failures := 0
+		skipped := 0
+		caseNo := 0
+		tsTime := time.Second * 0
+		suiteNo++
+
+		oTs := Testsuite{
+			Text:     ts.Description,
+			ID:       suiteNo,
+			Name:     ts.Name,
+			Time:     "",
+			Testcase: []Testcase{},
+		}
+
+		for _, tc := range ts.TestCases {
+			caseNo++
+			testSkipped := false
+
+			tcTime := tc.End.Sub(tc.Start)
+			tsTime += tcTime
+
+			oTc := Testcase{
+				Text: tc.Description,
+				ID:   caseNo,
+				Name: tc.Name,
+				Time: fmt.Sprintf("%v", tcTime.Seconds()),
+			}
+
+			if contains(exclusions, ts.Name, tc.Name) {
+				testSkipped = true
+			}
+
+			if !tc.SummaryResult.Pass && !testSkipped {
+				failures++
+				oTc.Failure = &Failure{
+					Text:    tc.SummaryResult.Details,
+					Message: "Error",
+					Type:    "ERROR",
+				}
+			}
+
+			// only skip on failure, if we pass, might as well include in results
+			if !tc.SummaryResult.Pass && testSkipped {
+				skipped++
+				log15.Info("test skipped", "name", tc.Name)
+				oTc.Skipped = &Skipped{
+					Text:    tc.SummaryResult.Details,
+					Message: "Skipped",
+					Type:    "SKIPPED",
+				}
+			}
+
+			tests++
+			oTs.Testcase = append(oTs.Testcase, oTc)
+			oTs.Tests = tests
+			oTs.Failures = failures
+			oTs.Skips = skipped
+			oTs.Time = fmt.Sprintf("%v", tsTime.Seconds())
+		}
+
+		opTs.Testsuite = append(opTs.Testsuite, oTs)
+
+		totalTests += tests
+		totalFailures += failures
+		totalSkipped += skipped
+	}
+
+	opTs.Tests = totalTests
+	opTs.Failures = totalFailures
+	opTs.Skips = totalSkipped
+	opTs.ID = "0"
+	opTs.Name = "Hive Test Run"
+	opTs.Time = ""
+
+	xmlContent, _ := xml.MarshalIndent(opTs, "", " ")
+	xmlContent = []byte(xml.Header + string(xmlContent))
+	err := ioutil.WriteFile(filepath.Join(path, "output.xml"), xmlContent, 0644)
+	if err != nil {
+		log15.Error(err.Error())
+	}
+
+	testsRun := totalTests > 0
+
+	if totalFailures > 0 {
+		return false, testsRun
+	}
+	return true, testsRun
+}
+
+func collectOutput(ts *libhive.TestSuite, outputs *[]*libhive.TestSuite) {
+	*outputs = append(*outputs, ts)
+}
+
+type parseTs func(*libhive.TestSuite, *[]*libhive.TestSuite)
+
+func walkSummaryFiles(fsys fs.FS, dir string, parse parseTs, ts *[]*libhive.TestSuite) error {
+	logfiles, err := fs.ReadDir(fsys, dir)
+	if err != nil {
+		return err
+	}
+	// Sort by name newest-first.
+	sort.Slice(logfiles, func(i, j int) bool {
+		return logfiles[i].Name() > logfiles[j].Name()
+	})
+
+	for _, entry := range logfiles {
+		name := entry.Name()
+		if entry.IsDir() || !strings.HasSuffix(name, ".json") || skipFile(name) {
+			continue
+		}
+		suite, _ := parseSuite(fsys, filepath.Join(dir, name))
+		if suite != nil {
+			parse(suite, ts)
+		}
+	}
+	return nil
+}
+
+func parseSuite(fsys fs.FS, path string) (*libhive.TestSuite, fs.FileInfo) {
+	file, err := fsys.Open(path)
+	if err != nil {
+		log.Printf("Can't access summary file: %s", err)
+		return nil, nil
+	}
+	defer file.Close()
+
+	fileInfo, err := file.Stat()
+	if err != nil {
+		log.Printf("Can't access summary file: %s", err)
+		return nil, nil
+	}
+
+	var info libhive.TestSuite
+	if err := json.NewDecoder(file).Decode(&info); err != nil {
+		log.Printf("Skipping invalid summary file %s: %v", fileInfo.Name(), err)
+		return nil, nil
+	}
+	if !suiteValid(&info) {
+		log.Printf("Skipping invalid summary file %s", fileInfo.Name())
+		return nil, nil
+	}
+	return &info, fileInfo
+}
+
+func suiteValid(s *libhive.TestSuite) bool {
+	return s.SimulatorLog != ""
+}
+
+func skipFile(f string) bool {
+	return f == "errorReport.json" || f == "containerErrorReport.json" || strings.HasPrefix(f, ".")
+}
+
+func contains(ex *Exclusions, tsName, tcName string) bool {
+	for _, ts := range ex.TestSuites {
+		if tsName == ts.Name {
+			for _, tc := range ts.TestCases {
+				if removeImageName(tcName) == tc {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func removeImageName(s string) string {
+	// regex to remove the last bracketed string (image name appended to test name)
+	reg := regexp.MustCompile(`\([^)]*\)$`)
+	return strings.Trim(reg.ReplaceAllString(s, ""), " ")
+}

hive.go

@@ -32,6 +32,7 @@ func main() {
 		simLogLevel           = flag.Int("sim.loglevel", 3, "Selects log `level` of client instances. Supports values 0-5.")
 		simDevMode            = flag.Bool("dev", false, "Only starts the simulator API endpoint (listening at 127.0.0.1:3000 by default) without starting any simulators.")
 		simDevModeAPIEndpoint = flag.String("dev.addr", "127.0.0.1:3000", "Endpoint that the simulator API listens on")
+		errorOnFailingTests   = flag.Bool("exit.fail", true, "Exit with error code 1 if any test fails")
 
 		clients = flag.String("client", "go-ethereum", "Comma separated `list` of clients to use. Client names in the list may be given as\n"+
 			"just the client name, or a client_branch specifier. If a branch name is supplied,\n"+
@@ -130,12 +131,14 @@ func main() {
 		log15.Info(fmt.Sprintf("simulation %s finished", sim), "suites", result.Suites, "tests", result.Tests, "failed", result.TestsFailed)
 	}
 
-	switch failCount {
-	case 0:
-	case 1:
-		fatal(errors.New("1 test failed"))
-	default:
-		fatal(fmt.Errorf("%d tests failed", failCount))
+	if *errorOnFailingTests && failCount > 0 {
+		switch failCount {
+		case 0:
+		case 1:
+			fatal(errors.New("1 test failed"))
+		default:
+			fatal(fmt.Errorf("%d tests failed", failCount))
+		}
 	}
 }