Skip to content

Commit d15d078

Browse files
yroblayrobla
authored
Merge pull request #7 from stacklok/add-maven
feat: add support for java
2 parents fa597ca + 1ed83e8 commit d15d078

File tree

5 files changed

+275
-1
lines changed

5 files changed

+275
-1
lines changed

‎pkg/config/scheduledfeed.go

+7
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/ossf/package-feeds/pkg/feeds"
1818
"github.com/ossf/package-feeds/pkg/feeds/crates"
1919
"github.com/ossf/package-feeds/pkg/feeds/goproxy"
20+
"github.com/ossf/package-feeds/pkg/feeds/maven"
2021
"github.com/ossf/package-feeds/pkg/feeds/npm"
2122
"github.com/ossf/package-feeds/pkg/feeds/nuget"
2223
"github.com/ossf/package-feeds/pkg/feeds/packagist"
@@ -179,6 +180,8 @@ func (fc FeedConfig) ToFeed(eventHandler *events.Handler) (feeds.ScheduledFeed,
179180
return npm.New(fc.Options, eventHandler)
180181
case nuget.FeedName:
181182
return nuget.New(fc.Options)
183+
case maven.FeedName:
184+
return maven.New(fc.Options)
182185
case pypi.FeedName:
183186
return pypi.New(fc.Options, eventHandler)
184187
case packagist.FeedName:
@@ -222,6 +225,10 @@ func Default() *ScheduledFeedConfig {
222225
Type: nuget.FeedName,
223226
Options: defaultFeedOptions,
224227
},
228+
{
229+
Type: maven.FeedName,
230+
Options: defaultFeedOptions,
231+
},
225232
{
226233
Type: packagist.FeedName,
227234
Options: defaultFeedOptions,

‎pkg/feeds/feed.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func NewArtifact(created time.Time, name, version, artifactID, feed string) *Pac
7070
func ApplyCutoff(pkgs []*Package, cutoff time.Time) []*Package {
7171
filteredPackages := []*Package{}
7272
for _, pkg := range pkgs {
73-
if pkg.CreatedDate.After(cutoff) {
73+
if pkg.CreatedDate.UTC().After(cutoff) {
7474
filteredPackages = append(filteredPackages, pkg)
7575
}
7676
}

‎pkg/feeds/maven/README.md

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# maven Feed
2+
3+
This feed allows polling of package updates from central.sonatype, polling Maven central repository.
4+
5+
## Configuration options
6+
7+
The `packages` field is not supported by the maven feed.
8+
9+
10+
```
11+
feeds:
12+
- type: maven
13+
```

‎pkg/feeds/maven/maven.go

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
package maven
2+
3+
import (
4+
"bytes"
5+
"encoding/json"
6+
"fmt"
7+
"net/http"
8+
"time"
9+
10+
"github.com/ossf/package-feeds/pkg/feeds"
11+
)
12+
13+
const (
14+
FeedName = "maven"
15+
indexPath = "/api/internal/browse/components"
16+
)
17+
18+
type Feed struct {
19+
baseURL string
20+
options feeds.FeedOptions
21+
}
22+
23+
func New(feedOptions feeds.FeedOptions) (*Feed, error) {
24+
if feedOptions.Packages != nil {
25+
return nil, feeds.UnsupportedOptionError{
26+
Feed: FeedName,
27+
Option: "packages",
28+
}
29+
}
30+
return &Feed{
31+
baseURL: "https://central.sonatype.com/" + indexPath,
32+
options: feedOptions,
33+
}, nil
34+
}
35+
36+
// Package represents package information.
37+
type LatestVersionInfo struct {
38+
Version string `json:"version"`
39+
TimestampUnixWithMS int64 `json:"timestampUnixWithMS"`
40+
}
41+
42+
type Package struct {
43+
Name string `json:"name"`
44+
Namespace string `json:"namespace"`
45+
LatestVersionInfo LatestVersionInfo `json:"latestVersionInfo"`
46+
}
47+
48+
// Response represents the response structure from Sonatype API.
49+
type Response struct {
50+
Components []Package `json:"components"`
51+
}
52+
53+
// fetchPackages fetches packages from Sonatype API for the given page.
54+
func (feed Feed) fetchPackages(page int) ([]Package, error) {
55+
// Define the request payload
56+
payload := map[string]interface{}{
57+
"page": page,
58+
"size": 20,
59+
"sortField": "publishedDate",
60+
"sortDirection": "desc",
61+
}
62+
63+
jsonPayload, err := json.Marshal(payload)
64+
if err != nil {
65+
return nil, fmt.Errorf("error encoding JSON: %w", err)
66+
}
67+
68+
// Send POST request to Sonatype API.
69+
resp, err := http.Post(feed.baseURL+"?repository=maven-central", "application/json", bytes.NewBuffer(jsonPayload))
70+
if err != nil {
71+
return nil, fmt.Errorf("error sending request: %w", err)
72+
}
73+
defer resp.Body.Close()
74+
75+
// Handle rate limiting (HTTP status code 429).
76+
if resp.StatusCode == http.StatusTooManyRequests {
77+
time.Sleep(5 * time.Second)
78+
return feed.fetchPackages(page) // Retry the request
79+
}
80+
81+
// Decode response.
82+
var response Response
83+
err = json.NewDecoder(resp.Body).Decode(&response)
84+
if err != nil {
85+
return nil, fmt.Errorf("error decoding response: %w", err)
86+
}
87+
return response.Components, nil
88+
}
89+
90+
func (feed Feed) Latest(cutoff time.Time) ([]*feeds.Package, time.Time, []error) {
91+
pkgs := []*feeds.Package{}
92+
var errs []error
93+
94+
page := 0
95+
for {
96+
// Fetch packages from Sonatype API for the current page.
97+
packages, err := feed.fetchPackages(page)
98+
if err != nil {
99+
errs = append(errs, err)
100+
break
101+
}
102+
103+
// Iterate over packages
104+
hasToCut := false
105+
for _, pkg := range packages {
106+
// convert published to date to compare with cutoff
107+
if pkg.LatestVersionInfo.TimestampUnixWithMS > cutoff.UnixMilli() {
108+
// Append package to pkgs
109+
timestamp := time.Unix(pkg.LatestVersionInfo.TimestampUnixWithMS/1000, 0)
110+
packageName := pkg.Namespace + ":" + pkg.Name
111+
112+
newPkg := feeds.NewPackage(timestamp, packageName, pkg.LatestVersionInfo.Version, FeedName)
113+
pkgs = append(pkgs, newPkg)
114+
} else {
115+
// Break the loop if the cutoff date is reached
116+
hasToCut = true
117+
}
118+
}
119+
120+
// Move to the next page
121+
page++
122+
123+
// Check if the loop should be terminated
124+
if len(pkgs) == 0 || hasToCut {
125+
break
126+
}
127+
}
128+
129+
newCutoff := feeds.FindCutoff(cutoff, pkgs)
130+
pkgs = feeds.ApplyCutoff(pkgs, cutoff)
131+
132+
return pkgs, newCutoff, errs
133+
}
134+
135+
func (feed Feed) GetName() string {
136+
return FeedName
137+
}
138+
139+
func (feed Feed) GetFeedOptions() feeds.FeedOptions {
140+
return feed.options
141+
}

‎pkg/feeds/maven/maven_test.go

+113
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package maven
2+
3+
import (
4+
"net/http"
5+
"testing"
6+
"time"
7+
8+
"github.com/ossf/package-feeds/pkg/feeds"
9+
testutils "github.com/ossf/package-feeds/pkg/utils/test"
10+
)
11+
12+
func TestMavenLatest(t *testing.T) {
13+
t.Parallel()
14+
15+
handlers := map[string]testutils.HTTPHandlerFunc{
16+
indexPath: mavenPackageResponse,
17+
}
18+
srv := testutils.HTTPServerMock(handlers)
19+
20+
feed, err := New(feeds.FeedOptions{})
21+
if err != nil {
22+
t.Fatalf("Failed to create Maven feed: %v", err)
23+
}
24+
feed.baseURL = srv.URL + "/api/internal/browse/components"
25+
26+
cutoff := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC)
27+
pkgs, gotCutoff, errs := feed.Latest(cutoff)
28+
29+
if len(errs) != 0 {
30+
t.Fatalf("feed.Latest returned error: %v", err)
31+
}
32+
33+
// Returned cutoff should match the newest package creation time of packages retrieved.
34+
wantCutoff := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
35+
if gotCutoff.UTC().Sub(wantCutoff).Abs() > time.Second {
36+
t.Errorf("Latest() cutoff %v, want %v", gotCutoff, wantCutoff)
37+
}
38+
if pkgs[0].Name != "com.github.example:project" {
39+
t.Errorf("Unexpected package `%s` found in place of expected `com.github.example:project`", pkgs[0].Name)
40+
}
41+
if pkgs[0].Version != "1.0.0" {
42+
t.Errorf("Unexpected version `%s` found in place of expected `1.0.0`", pkgs[0].Version)
43+
}
44+
45+
for _, p := range pkgs {
46+
if p.Type != FeedName {
47+
t.Errorf("Feed type not set correctly in goproxy package following Latest()")
48+
}
49+
}
50+
}
51+
52+
func TestMavenNotFound(t *testing.T) {
53+
t.Parallel()
54+
55+
handlers := map[string]testutils.HTTPHandlerFunc{
56+
indexPath: testutils.NotFoundHandlerFunc,
57+
}
58+
srv := testutils.HTTPServerMock(handlers)
59+
60+
feed, err := New(feeds.FeedOptions{})
61+
if err != nil {
62+
t.Fatalf("Failed to create Maven feed: %v", err)
63+
}
64+
feed.baseURL = srv.URL + "/api/internal/browse/components"
65+
66+
cutoff := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
67+
68+
_, gotCutoff, errs := feed.Latest(cutoff)
69+
if cutoff != gotCutoff {
70+
t.Error("feed.Latest() cutoff should be unchanged if an error is returned")
71+
}
72+
if len(errs) == 0 {
73+
t.Fatalf("feed.Latest() was successful when an error was expected")
74+
}
75+
}
76+
77+
func mavenPackageResponse(w http.ResponseWriter, r *http.Request) {
78+
w.Header().Set("Content-Type", "application/json")
79+
responseJSON := `
80+
{
81+
"components": [
82+
{
83+
"id": "pkg:maven/com.github.example/project",
84+
"type": "COMPONENT",
85+
"namespace": "com.github.example",
86+
"name": "project",
87+
"version": "1.0.0",
88+
"publishedEpochMillis": 946684800000,
89+
"latestVersionInfo": {
90+
"version": "1.0.0",
91+
"timestampUnixWithMS": 946684800000
92+
}
93+
},
94+
{
95+
"id": "pkg:maven/com.github.example/project1",
96+
"type": "COMPONENT",
97+
"namespace": "com.github.example",
98+
"name": "project",
99+
"version": "1.0.0",
100+
"publishedEpochMillis": null,
101+
"latestVersionInfo": {
102+
"version": "1.0.0",
103+
"timestampUnixWithMS": 0
104+
}
105+
}
106+
]
107+
}
108+
`
109+
_, err := w.Write([]byte(responseJSON))
110+
if err != nil {
111+
http.Error(w, testutils.UnexpectedWriteError(err), http.StatusInternalServerError)
112+
}
113+
}

0 commit comments

Comments
 (0)