Skip to content

Commit 8713bac

Browse files
unknwonmpsonntag
unknwon
authored and
mpsonntag
committed
db: use GORM to backup and restore non-legacy tables (#6142)
1 parent 3b36d0c commit 8713bac

25 files changed

+592
-187
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ All notable changes to Gogs are documented in this file.
4747
- [Security] Potential RCE on mirror repositories. [#5767](https://github.com/gogs/gogs/issues/5767)
4848
- [Security] Potential XSS attack with raw markdown API. [#5907](https://github.com/gogs/gogs/pull/5907)
4949
- File both modified and renamed within a commit treated as separate files. [#5056](https://github.com/gogs/gogs/issues/5056)
50+
- Unable to restore the database backup to MySQL 8.0 with syntax error. [#5602](https://github.com/gogs/gogs/issues/5602)
5051
- Open/close milestone redirects to a 404 page. [#5677](https://github.com/gogs/gogs/issues/5677)
5152
- Disallow multiple tokens with same name. [#5587](https://github.com/gogs/gogs/issues/5587) [#5820](https://github.com/gogs/gogs/pull/5820)
5253
- Enable Federated Avatar Lookup could cause server to crash. [#5848](https://github.com/gogs/gogs/issues/5848)

‎internal/cmd/admin.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ func runCreateUser(c *cli.Context) error {
148148
}
149149
conf.InitLogging(true)
150150

151-
if err = db.SetEngine(); err != nil {
151+
if _, err = db.SetEngine(); err != nil {
152152
return errors.Wrap(err, "set engine")
153153
}
154154

@@ -174,7 +174,7 @@ func adminDashboardOperation(operation func() error, successMessage string) func
174174
}
175175
conf.InitLogging(true)
176176

177-
if err = db.SetEngine(); err != nil {
177+
if _, err = db.SetEngine(); err != nil {
178178
return errors.Wrap(err, "set engine")
179179
}
180180

‎internal/cmd/backup.go

+12-11
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ portable among all supported database engines.`,
4242
},
4343
}
4444

45-
const _CURRENT_BACKUP_FORMAT_VERSION = 1
46-
const _ARCHIVE_ROOT_DIR = "gogs-backup"
45+
const currentBackupFormatVersion = 1
46+
const archiveRootDir = "gogs-backup"
4747

4848
func runBackup(c *cli.Context) error {
4949
zip.Verbose = c.Bool("verbose")
@@ -54,7 +54,8 @@ func runBackup(c *cli.Context) error {
5454
}
5555
conf.InitLogging(true)
5656

57-
if err = db.SetEngine(); err != nil {
57+
conn, err := db.SetEngine()
58+
if err != nil {
5859
return errors.Wrap(err, "set engine")
5960
}
6061

@@ -71,7 +72,7 @@ func runBackup(c *cli.Context) error {
7172
// Metadata
7273
metaFile := path.Join(rootDir, "metadata.ini")
7374
metadata := ini.Empty()
74-
metadata.Section("").Key("VERSION").SetValue(com.ToStr(_CURRENT_BACKUP_FORMAT_VERSION))
75+
metadata.Section("").Key("VERSION").SetValue(com.ToStr(currentBackupFormatVersion))
7576
metadata.Section("").Key("DATE_TIME").SetValue(time.Now().String())
7677
metadata.Section("").Key("GOGS_VERSION").SetValue(conf.App.Version)
7778
if err = metadata.SaveTo(metaFile); err != nil {
@@ -85,22 +86,22 @@ func runBackup(c *cli.Context) error {
8586
if err != nil {
8687
log.Fatal("Failed to create backup archive '%s': %v", archiveName, err)
8788
}
88-
if err = z.AddFile(_ARCHIVE_ROOT_DIR+"/metadata.ini", metaFile); err != nil {
89+
if err = z.AddFile(archiveRootDir+"/metadata.ini", metaFile); err != nil {
8990
log.Fatal("Failed to include 'metadata.ini': %v", err)
9091
}
9192

9293
// Database
9394
dbDir := filepath.Join(rootDir, "db")
94-
if err = db.DumpDatabase(dbDir); err != nil {
95+
if err = db.DumpDatabase(conn, dbDir, c.Bool("verbose")); err != nil {
9596
log.Fatal("Failed to dump database: %v", err)
9697
}
97-
if err = z.AddDir(_ARCHIVE_ROOT_DIR+"/db", dbDir); err != nil {
98+
if err = z.AddDir(archiveRootDir+"/db", dbDir); err != nil {
9899
log.Fatal("Failed to include 'db': %v", err)
99100
}
100101

101102
// Custom files
102103
if !c.Bool("database-only") {
103-
if err = z.AddDir(_ARCHIVE_ROOT_DIR+"/custom", conf.CustomDir()); err != nil {
104+
if err = z.AddDir(archiveRootDir+"/custom", conf.CustomDir()); err != nil {
104105
log.Fatal("Failed to include 'custom': %v", err)
105106
}
106107
}
@@ -113,7 +114,7 @@ func runBackup(c *cli.Context) error {
113114
continue
114115
}
115116

116-
if err = z.AddDir(path.Join(_ARCHIVE_ROOT_DIR+"/data", dir), dirPath); err != nil {
117+
if err = z.AddDir(path.Join(archiveRootDir+"/data", dir), dirPath); err != nil {
117118
log.Fatal("Failed to include 'data': %v", err)
118119
}
119120
}
@@ -149,7 +150,7 @@ func runBackup(c *cli.Context) error {
149150
}
150151
log.Info("Repositories dumped to: %s", reposDump)
151152

152-
if err = z.AddFile(_ARCHIVE_ROOT_DIR+"/repositories.zip", reposDump); err != nil {
153+
if err = z.AddFile(archiveRootDir+"/repositories.zip", reposDump); err != nil {
153154
log.Fatal("Failed to include %q: %v", reposDump, err)
154155
}
155156
}
@@ -158,7 +159,7 @@ func runBackup(c *cli.Context) error {
158159
log.Fatal("Failed to save backup archive '%s': %v", archiveName, err)
159160
}
160161

161-
os.RemoveAll(rootDir)
162+
_ = os.RemoveAll(rootDir)
162163
log.Info("Backup succeed! Archive is located at: %s", archiveName)
163164
log.Stop()
164165
return nil

‎internal/cmd/restore.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ func runRestore(c *cli.Context) error {
5757
if err := zip.ExtractTo(c.String("from"), tmpDir); err != nil {
5858
log.Fatal("Failed to extract backup archive: %v", err)
5959
}
60-
archivePath := path.Join(tmpDir, _ARCHIVE_ROOT_DIR)
60+
archivePath := path.Join(tmpDir, archiveRootDir)
6161
defer os.RemoveAll(archivePath)
6262

6363
// Check backup version
@@ -77,9 +77,9 @@ func runRestore(c *cli.Context) error {
7777
if formatVersion == 0 {
7878
log.Fatal("Failed to determine the backup format version from metadata '%s': %s", metaFile, "VERSION is not presented")
7979
}
80-
if formatVersion != _CURRENT_BACKUP_FORMAT_VERSION {
80+
if formatVersion != currentBackupFormatVersion {
8181
log.Fatal("Backup format version found is %d but this binary only supports %d\nThe last known version that is able to import your backup is %s",
82-
formatVersion, _CURRENT_BACKUP_FORMAT_VERSION, lastSupportedVersionOfFormat[formatVersion])
82+
formatVersion, currentBackupFormatVersion, lastSupportedVersionOfFormat[formatVersion])
8383
}
8484

8585
// If config file is not present in backup, user must set this file via flag.
@@ -100,13 +100,14 @@ func runRestore(c *cli.Context) error {
100100
}
101101
conf.InitLogging(true)
102102

103-
if err = db.SetEngine(); err != nil {
103+
conn, err := db.SetEngine()
104+
if err != nil {
104105
return errors.Wrap(err, "set engine")
105106
}
106107

107108
// Database
108109
dbDir := path.Join(archivePath, "db")
109-
if err = db.ImportDatabase(dbDir, c.Bool("verbose")); err != nil {
110+
if err = db.ImportDatabase(conn, dbDir, c.Bool("verbose")); err != nil {
110111
log.Fatal("Failed to import database: %v", err)
111112
}
112113

‎internal/cmd/serv.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ func setup(c *cli.Context, logPath string, connectDB bool) {
9393
_ = os.Chdir(conf.WorkDir())
9494
}
9595

96-
if err := db.SetEngine(); err != nil {
96+
if _, err := db.SetEngine(); err != nil {
9797
fail("Internal error", "Failed to set database engine: %v", err)
9898
}
9999
}

‎internal/cryptoutil/sha1.go

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright 2020 The Gogs Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cryptoutil
6+
7+
import (
8+
"crypto/sha1"
9+
"encoding/hex"
10+
)
11+
12+
// SHA1 encodes string to hexadecimal of SHA1 checksum.
13+
func SHA1(str string) string {
14+
h := sha1.New()
15+
_, _ = h.Write([]byte(str))
16+
return hex.EncodeToString(h.Sum(nil))
17+
}

‎internal/cryptoutil/sha1_test.go

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Copyright 2020 The Gogs Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cryptoutil
6+
7+
import (
8+
"testing"
9+
10+
"github.com/stretchr/testify/assert"
11+
)
12+
13+
func TestSHA1(t *testing.T) {
14+
tests := []struct {
15+
input string
16+
output string
17+
}{
18+
{input: "", output: "da39a3ee5e6b4b0d3255bfef95601890afd80709"},
19+
{input: "The quick brown fox jumps over the lazy dog", output: "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12"},
20+
{input: "The quick brown fox jumps over the lazy dog.", output: "408d94384216f890ff7a0c3528e8bed1e0b01621"},
21+
}
22+
for _, test := range tests {
23+
t.Run(test.input, func(t *testing.T) {
24+
assert.Equal(t, test.output, SHA1(test.input))
25+
})
26+
}
27+
}

‎internal/db/access_tokens.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ import (
1111
"github.com/jinzhu/gorm"
1212
gouuid "github.com/satori/go.uuid"
1313

14+
"github.com/G-Node/gogs/internal/cryptoutil"
1415
"github.com/G-Node/gogs/internal/errutil"
15-
"github.com/G-Node/gogs/internal/tool"
1616
)
1717

1818
// AccessTokensStore is the persistent interface for access tokens.
@@ -56,6 +56,9 @@ type AccessToken struct {
5656

5757
// NOTE: This is a GORM create hook.
5858
func (t *AccessToken) BeforeCreate() {
59+
if t.CreatedUnix > 0 {
60+
return
61+
}
5962
t.CreatedUnix = gorm.NowFunc().Unix()
6063
}
6164

@@ -102,7 +105,7 @@ func (db *accessTokens) Create(userID int64, name string) (*AccessToken, error)
102105
token := &AccessToken{
103106
UserID: userID,
104107
Name: name,
105-
Sha1: tool.SHA1(gouuid.NewV4().String()),
108+
Sha1: cryptoutil.SHA1(gouuid.NewV4().String()),
106109
}
107110
return token, db.DB.Create(token).Error
108111
}

‎internal/db/access_tokens_test.go

+18-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,22 @@ import (
1414
"github.com/G-Node/gogs/internal/errutil"
1515
)
1616

17+
func TestAccessToken_BeforeCreate(t *testing.T) {
18+
t.Run("CreatedUnix has been set", func(t *testing.T) {
19+
token := &AccessToken{CreatedUnix: 1}
20+
token.BeforeCreate()
21+
assert.Equal(t, int64(1), token.CreatedUnix)
22+
assert.Equal(t, int64(0), token.UpdatedUnix)
23+
})
24+
25+
t.Run("CreatedUnix has not been set", func(t *testing.T) {
26+
token := &AccessToken{}
27+
token.BeforeCreate()
28+
assert.Equal(t, gorm.NowFunc().Unix(), token.CreatedUnix)
29+
assert.Equal(t, int64(0), token.UpdatedUnix)
30+
})
31+
}
32+
1733
func Test_accessTokens(t *testing.T) {
1834
if testing.Short() {
1935
t.Skip()
@@ -64,7 +80,7 @@ func test_accessTokens_Create(t *testing.T, db *accessTokens) {
6480
if err != nil {
6581
t.Fatal(err)
6682
}
67-
assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), token.Created.Format(time.RFC3339))
83+
assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), token.Created.UTC().Format(time.RFC3339))
6884

6985
// Try create second access token with same name should fail
7086
_, err = db.Create(token.UserID, token.Name)
@@ -177,5 +193,5 @@ func test_accessTokens_Save(t *testing.T, db *accessTokens) {
177193
if err != nil {
178194
t.Fatal(err)
179195
}
180-
assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), token.Updated.Format(time.RFC3339))
196+
assert.Equal(t, gorm.NowFunc().Format(time.RFC3339), token.Updated.UTC().Format(time.RFC3339))
181197
}

0 commit comments

Comments
 (0)