Skip to content

Commit 6e6b2d6

Browse files
voidintvoidint
authored
Merge pull request #138 from 863056768/master
feat: support USTC mirror site
2 parents 91c8196 + 25bebd9 commit 6e6b2d6

12 files changed

+25490
-14
lines changed

‎cli/init.go

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package cli
2+
3+
import (
4+
_ "github.com/voidint/g/collector/aliyun"
5+
_ "github.com/voidint/g/collector/ustc"
6+
)

‎collector/aliyun/aliyun_collector.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"strings"
88

99
"github.com/PuerkitoBio/goquery"
10+
"github.com/voidint/g/collector"
1011
"github.com/voidint/g/pkg/errs"
1112
httppkg "github.com/voidint/g/pkg/http"
1213
"github.com/voidint/g/version"
@@ -16,9 +17,12 @@ import (
1617

1718
const (
1819
// DownloadPageURL 阿里云镜像站点网址
19-
DownloadPageURL = "https://mirrors.aliyun.com/golang/"
20+
DownloadPageDomain = "mirrors.aliyun.com"
21+
DownloadPageURL = "https://" + DownloadPageDomain + "/golang/"
2022
)
2123

24+
func init() { collector.Register(DownloadPageDomain, NewCollector) }
25+
2226
// Collector 阿里云镜像站点版本采集器
2327
type Collector struct {
2428
url string
@@ -27,7 +31,7 @@ type Collector struct {
2731
}
2832

2933
// NewCollector 返回采集器实例
30-
func NewCollector() (*Collector, error) {
34+
func NewCollector() (collector.Collector, error) {
3135
pURL, err := stdurl.Parse(DownloadPageURL)
3236
if err != nil {
3337
return nil, err

‎collector/aliyun/aliyun_collector_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,8 @@ func TestNewCollector(t *testing.T) {
126126
got, err := NewCollector()
127127
assert.Equal(t, tt.wantErr, err)
128128
if err == nil {
129-
assert.NotNil(t, got.pURL)
130-
assert.NotNil(t, got.doc)
129+
assert.NotNil(t, got.(*Collector).pURL)
130+
assert.NotNil(t, got.(*Collector).doc)
131131
}
132132
})
133133
}

‎collector/aliyun/go_file_item_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func Test_getGoVersion(t *testing.T) {
3333
In: &goFileItem{
3434
FileName: "go1.18rc1.darwin-amd64.tar.gz",
3535
URL: "https://mirrors.aliyun.com/golang/go1.18rc1.darwin-amd64.tar.gz",
36-
Size: "107.2 MB",
36+
Size: "137.0 MB",
3737
},
3838
Expected: "1.18rc1",
3939
},

‎collector/collector.go

+19-7
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,21 @@
11
package collector
22

33
import (
4+
stdurl "net/url"
45
"strings"
56

6-
"github.com/voidint/g/collector/aliyun"
77
"github.com/voidint/g/collector/official"
88
"github.com/voidint/g/version"
99
)
1010

11+
var collectors = make(map[string]Builder)
12+
13+
func Register(domain string, b Builder) {
14+
collectors[domain] = b
15+
}
16+
17+
type Builder func() (Collector, error)
18+
1119
// Collector 版本信息采集器
1220
type Collector interface {
1321
// 返回稳定版本列表
@@ -25,16 +33,20 @@ func NewCollector(urls ...string) (c Collector, err error) {
2533
if len(urls) == 0 {
2634
urls = []string{official.DefaultDownloadPageURL}
2735
}
28-
for i := range urls {
29-
urls[i] = strings.TrimSpace(urls[i])
36+
for _, rawUrl := range urls {
37+
var url *stdurl.URL
38+
url, err = stdurl.Parse(strings.TrimSpace(rawUrl))
39+
if err != nil {
40+
continue
41+
}
3042

31-
if urls[i] != "" && (strings.HasPrefix(aliyun.DownloadPageURL, urls[i]) || strings.HasPrefix(urls[i], aliyun.DownloadPageURL)) {
32-
if c, err = aliyun.NewCollector(); err == nil {
33-
return c, nil
43+
for domain, c := range collectors {
44+
if url.Host == strings.ToLower(domain) {
45+
return c()
3446
}
3547
}
3648

37-
if c, err = official.NewCollector(urls[i]); err == nil {
49+
if c, err = official.NewCollector(rawUrl); err == nil {
3850
return c, nil
3951
}
4052
}

‎collector/ustc/go_file_item.go

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
package ustc
2+
3+
import (
4+
"fmt"
5+
"sort"
6+
"strings"
7+
"unicode"
8+
9+
"github.com/voidint/g/pkg/checksum"
10+
"github.com/voidint/g/version"
11+
)
12+
13+
type goFileItem struct {
14+
FileName string
15+
URL string
16+
Size string
17+
}
18+
19+
func (item goFileItem) getGoVersion() string {
20+
arr := strings.Split(strings.TrimPrefix(item.FileName, "go"), ".")
21+
if len(arr) < 3 || !unicode.IsNumber(rune(arr[0][0])) {
22+
return ""
23+
}
24+
if unicode.IsNumber(rune(arr[2][0])) {
25+
return fmt.Sprintf("%s.%s.%s", arr[0], arr[1], arr[2])
26+
}
27+
28+
if unicode.IsNumber(rune(arr[1][0])) {
29+
return fmt.Sprintf("%s.%s", arr[0], arr[1])
30+
}
31+
return arr[0]
32+
}
33+
34+
func (item goFileItem) isSHA256File() bool {
35+
return strings.HasSuffix(item.FileName, ".sha256")
36+
}
37+
38+
func (item goFileItem) isPackageFile() bool {
39+
return strings.HasSuffix(item.FileName, ".tar.gz") ||
40+
strings.HasSuffix(item.FileName, ".pkg") ||
41+
strings.HasSuffix(item.FileName, ".zip") ||
42+
strings.HasSuffix(item.FileName, ".msi")
43+
}
44+
45+
func (item goFileItem) getKind() version.PackageKind {
46+
if strings.HasSuffix(item.FileName, ".src.tar.gz") {
47+
return version.SourceKind
48+
}
49+
if strings.HasSuffix(item.FileName, ".tar.gz") || strings.HasSuffix(item.FileName, ".zip") {
50+
return version.ArchiveKind
51+
}
52+
if strings.HasSuffix(item.FileName, ".pkg") || strings.HasSuffix(item.FileName, ".msi") {
53+
return version.InstallerKind
54+
}
55+
return "Unknown"
56+
}
57+
58+
var osMapping = map[string]string{
59+
"linux": "Linux",
60+
"darwin": "macOS",
61+
"windows": "Windows",
62+
"freebsd": "FreeBSD",
63+
"netbsd": "netbsd",
64+
"openbsd": "openbsd",
65+
"solaris": "solaris",
66+
"plan9": "plan9",
67+
"aix": "aix",
68+
"dragonfly": "dragonfly",
69+
"illumos": "illumos",
70+
}
71+
72+
func (item goFileItem) getOS() string {
73+
for k, v := range osMapping {
74+
if strings.Contains(item.FileName, k) {
75+
return v
76+
}
77+
}
78+
return ""
79+
}
80+
81+
var archMapping = map[string]string{
82+
"-386.": "x86",
83+
"-amd64.": "x86-64",
84+
"-arm.": "ARMv6",
85+
"-arm64.": "ARM64",
86+
"-armv6l.": "ARMv6",
87+
"-ppc64.": "ppc64",
88+
"-ppc64le.": "ppc64le",
89+
"-mips.": "mips",
90+
"-mipsle.": "mipsle",
91+
"-mips64.": "mips64",
92+
"-mips64le.": "mips64le",
93+
"-s390x.": "s390x",
94+
"-riscv64.": "riscv64",
95+
"-loong64.": "loong64",
96+
}
97+
98+
func (item goFileItem) getArch() string {
99+
for k, v := range archMapping {
100+
if strings.Contains(item.FileName, k) {
101+
return v
102+
}
103+
}
104+
return ""
105+
}
106+
107+
func convert2Versions(items []*goFileItem) (vers []*version.Version, err error) {
108+
pkgMap := make(map[string][]*version.Package, 20)
109+
110+
for _, pitem := range items {
111+
ver := pitem.getGoVersion()
112+
if _, ok := pkgMap[ver]; !ok {
113+
pkgMap[ver] = make([]*version.Package, 0, 20)
114+
}
115+
116+
if pitem.isPackageFile() {
117+
pkgMap[ver] = append(pkgMap[ver], &version.Package{
118+
FileName: pitem.FileName,
119+
URL: pitem.URL,
120+
Kind: pitem.getKind(),
121+
OS: pitem.getOS(),
122+
Arch: pitem.getArch(),
123+
Size: pitem.Size,
124+
})
125+
} else if pitem.isSHA256File() {
126+
// 设置校验和及算法
127+
for _, ppkg := range pkgMap[ver] {
128+
if !strings.HasPrefix(pitem.FileName, ppkg.FileName) {
129+
continue
130+
}
131+
ppkg.Algorithm = string(checksum.SHA256)
132+
ppkg.ChecksumURL = pitem.URL
133+
}
134+
}
135+
}
136+
137+
vers = make([]*version.Version, 0, len(pkgMap))
138+
for vname, pkgs := range pkgMap {
139+
v, err := version.New(vname, version.WithPackages(pkgs))
140+
if err != nil {
141+
return nil, err
142+
}
143+
vers = append(vers, v)
144+
}
145+
sort.Sort(version.Collection(vers))
146+
return vers, nil
147+
}

0 commit comments

Comments
 (0)