From d5d303afb8c73fbca2a490e962a54dd26cc7adae Mon Sep 17 00:00:00 2001 From: Dan Rollo Date: Tue, 12 Feb 2019 14:09:31 -0500 Subject: [PATCH] CI nancy investigates nancy (#8) * run the just built binary against this project * move eval calls into 'script' section to fail build if eval fails * fix test bug due to stale test db files having expired TTL cache items. Also add new test of TTL expiration. * be certain the TTL expires by waiting * add myself to the list of the usual suspects. --- .travis.yml | 2 + CONTRIBUTORS.md | 1 + ossindex/ossindex.go | 13 +++++-- ossindex/ossindex_test.go | 78 +++++++++++++++++++++++++++++++-------- 4 files changed, 75 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d2315eb..1fb88762 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ script: - GOOS=linux GOARCH=amd64 go build -ldflags="-X 'github.com/sonatype-nexus-community/nancy/buildversion.BuildVersion=$VERSION' -X 'github.com/sonatype-nexus-community/nancy/buildversion.BuildTime=$time' -X 'github.com/sonatype-nexus-community/nancy/buildversion.BuildCommit=$TRAVIS_COMMIT'" -o nancy-linux.amd64-$VERSION - GOOS=darwin GOARCH=amd64 go build -ldflags="-X 'github.com/sonatype-nexus-community/nancy/buildversion.BuildVersion=$VERSION' -X 'github.com/sonatype-nexus-community/nancy/buildversion.BuildTime=$time' -X 'github.com/sonatype-nexus-community/nancy/buildversion.BuildCommit=$TRAVIS_COMMIT'" -o nancy-darwin.amd64-$VERSION - GOOS=windows GOARCH=amd64 go build -ldflags="-X 'github.com/sonatype-nexus-community/nancy/buildversion.BuildVersion=$VERSION' -X 'github.com/sonatype-nexus-community/nancy/buildversion.BuildTime=$time' -X 'github.com/sonatype-nexus-community/nancy/buildversion.BuildCommit=$TRAVIS_COMMIT'" -o nancy-windows.amd64-$VERSION.exe + - ./nancy-linux.amd64-$VERSION Gopkg.lock + - ./nancy-linux.amd64-$VERSION go.sum env: - GO111MODULE=on diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index a3807f1f..51fcdfb1 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -5,6 +5,7 @@ Sonatype internal people: * [@ken-duck](https://github.com/ken-duck/) (Ken Duck) * [@DarthHater](https://github.com/darthhater/) (Jeffry Hesse) * [@ajbrown](https://github.com/ajbrown) (A.J. Brown) +* [@bhamail](https://github.com/bhamail) (Dan Rollo) External contributors: diff --git a/ossindex/ossindex.go b/ossindex/ossindex.go index bcddc3fa..01cc62eb 100644 --- a/ossindex/ossindex.go +++ b/ossindex/ossindex.go @@ -54,6 +54,14 @@ func getOssIndexUrl() string { return ossIndexUrl } +func openDb(dbDir string) (db *badger.DB, err error) { + opts := badger.DefaultOptions + opts.Dir = dbDir + "/" + dbValueDirName + opts.ValueDir = dbDir + "/" + dbValueDirName + db, err = badger.Open(opts) + return +} + // AuditPackages will given a list of Package URLs, run an OSS Index audit func AuditPackages(purls []string) ([]types.Coordinate, error) { dbDir := getDatabaseDirectory() @@ -62,10 +70,7 @@ func AuditPackages(purls []string) ([]types.Coordinate, error) { } // Initialize the cache - opts := badger.DefaultOptions - opts.Dir = dbDir + "/" + dbValueDirName - opts.ValueDir = dbDir + "/" + dbValueDirName - db, err := badger.Open(opts) + db, err := openDb(dbDir) customerrors.Check(err, "Error initializing cache") defer func() { diff --git a/ossindex/ossindex_test.go b/ossindex/ossindex_test.go index 1508afb9..d0f3ab32 100644 --- a/ossindex/ossindex_test.go +++ b/ossindex/ossindex_test.go @@ -18,6 +18,7 @@ package ossindex import ( "encoding/json" "fmt" + "github.com/dgraph-io/badger" "github.com/sonatype-nexus-community/nancy/types" "github.com/stretchr/testify/assert" "io" @@ -29,6 +30,7 @@ import ( "path" "strings" "testing" + "time" ) const purl = "pkg:github/BurntSushi/toml@0.3.1" @@ -165,20 +167,7 @@ func TestAuditPackages_ErrorBadResponseBody(t *testing.T) { func TestAuditPackages_NewPackage(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, http.MethodPost, r.Method) - assert.Equal(t, "/", r.URL.EscapedPath()) - - w.WriteHeader(http.StatusOK) - - coordinates := []types.Coordinate{ - { - Coordinates: "pkg:github/burntsushi/toml@0.3.1", - Reference: "https://ossindex.sonatype.org/component/pkg:github/burntsushi/toml@0.3.1", - Vulnerabilities: []types.Vulnerability{}, - }, - } - jsonCoordinates, _ := json.Marshal(coordinates) - _, _ = w.Write(jsonCoordinates) + verifyClientCallAndWriteValidPackageResponse(t, r, w) })) defer ts.Close() ossIndexUrl = ts.URL @@ -192,6 +181,21 @@ func TestAuditPackages_NewPackage(t *testing.T) { assert.Nil(t, err) } +func verifyClientCallAndWriteValidPackageResponse(t *testing.T, r *http.Request, w http.ResponseWriter) { + assert.Equal(t, http.MethodPost, r.Method) + assert.Equal(t, "/", r.URL.EscapedPath()) + w.WriteHeader(http.StatusOK) + coordinates := []types.Coordinate{ + { + Coordinates: "pkg:github/burntsushi/toml@0.3.1", + Reference: "https://ossindex.sonatype.org/component/pkg:github/burntsushi/toml@0.3.1", + Vulnerabilities: []types.Vulnerability{}, + }, + } + jsonCoordinates, _ := json.Marshal(coordinates) + _, _ = w.Write(jsonCoordinates) +} + // File copies a single file from src to dst func copyFile(src, dst string) error { var err error @@ -257,7 +261,7 @@ func copyDir(src string, dst string) error { func TestAuditPackages_SinglePackage_Cached(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Errorf("No call should occur with nil package. called: %v", r) + t.Errorf("No call should occur with previously cached package. called: %v", r) })) defer ts.Close() ossIndexUrl = ts.URL @@ -268,6 +272,50 @@ func TestAuditPackages_SinglePackage_Cached(t *testing.T) { // put test db cache dir in expected location cacheValueDir := getDatabaseDirectory() + "/" + dbValueDirName assert.Nil(t, copyDir("testdata/golang", cacheValueDir)) + // need to re-set the cached package to avoid test failures due to expiration of the TTL for the cached item + db, err := openDb(getDatabaseDirectory()) + assert.Nil(t, err) + assert.Nil(t, db.Update(func(txn *badger.Txn) error { + var coordJson, _ = json.Marshal(expectedCoordinate) + err := txn.SetWithTTL([]byte(strings.ToLower(lowerCasePurl)), []byte(coordJson), time.Hour*12) + if err != nil { + return err + } + return nil + })) + assert.Nil(t, db.Close()) + + coordinates, err := AuditPackages([]string{purl}) + assert.Equal(t, []types.Coordinate{expectedCoordinate}, coordinates) + assert.Nil(t, err) +} + +func TestAuditPackages_SinglePackage_Cached_WithExpiredTTL(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + verifyClientCallAndWriteValidPackageResponse(t, r, w) + })) + defer ts.Close() + ossIndexUrl = ts.URL + + teardownTestCase := setupTestCaseMoveCacheDb(t) + defer teardownTestCase(t) + + // put test db cache dir in expected location + cacheValueDir := getDatabaseDirectory() + "/" + dbValueDirName + assert.Nil(t, copyDir("testdata/golang", cacheValueDir)) + // need to re-set the cached package with short TTL for the cached item to ensure it expires before we read it + db, err := openDb(getDatabaseDirectory()) + assert.Nil(t, err) + assert.Nil(t, db.Update(func(txn *badger.Txn) error { + var coordJson, _ = json.Marshal(expectedCoordinate) + err := txn.SetWithTTL([]byte(strings.ToLower(lowerCasePurl)), []byte(coordJson), time.Second*1) + if err != nil { + return err + } + return nil + })) + assert.Nil(t, db.Close()) + time.Sleep(2 * time.Second) coordinates, err := AuditPackages([]string{purl}) assert.Equal(t, []types.Coordinate{expectedCoordinate}, coordinates)