aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cmd/main_test.go23
-rw-r--r--cmd/migrate_storage.go83
-rw-r--r--cmd/migrate_storage_test.go74
-rw-r--r--models/db/iterate.go34
-rw-r--r--models/git/lfs.go23
-rw-r--r--models/repo/attachment.go22
-rw-r--r--models/repo/repo_list.go24
-rw-r--r--models/user/search.go23
-rw-r--r--modules/packages/content_store.go10
-rw-r--r--routers/private/mail.go3
-rw-r--r--services/repository/avatar.go2
11 files changed, 187 insertions, 134 deletions
diff --git a/cmd/main_test.go b/cmd/main_test.go
new file mode 100644
index 0000000000..9cce0ef036
--- /dev/null
+++ b/cmd/main_test.go
@@ -0,0 +1,23 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+ "testing"
+
+ "code.gitea.io/gitea/models/unittest"
+ "code.gitea.io/gitea/modules/setting"
+)
+
+func init() {
+ setting.SetCustomPathAndConf("", "", "")
+ setting.LoadForTest()
+}
+
+func TestMain(m *testing.M) {
+ unittest.MainTest(m, &unittest.TestOptions{
+ GiteaRootPath: "..",
+ })
+}
diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go
index 93fb64a4d3..f11cf9b11f 100644
--- a/cmd/migrate_storage.go
+++ b/cmd/migrate_storage.go
@@ -12,9 +12,11 @@ import (
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
"code.gitea.io/gitea/models/migrations"
+ packages_model "code.gitea.io/gitea/models/packages"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/log"
+ packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
@@ -25,13 +27,13 @@ import (
var CmdMigrateStorage = cli.Command{
Name: "migrate-storage",
Usage: "Migrate the storage",
- Description: "This is a command for migrating storage.",
+ Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
Action: runMigrateStorage,
Flags: []cli.Flag{
cli.StringFlag{
Name: "type, t",
Value: "",
- Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
+ Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages'",
},
cli.StringFlag{
Name: "storage, s",
@@ -80,34 +82,53 @@ var CmdMigrateStorage = cli.Command{
},
}
-func migrateAttachments(dstStorage storage.ObjectStorage) error {
- return repo_model.IterateAttachment(func(attach *repo_model.Attachment) error {
+func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.IterateObjects(ctx, func(attach *repo_model.Attachment) error {
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
return err
})
}
-func migrateLFS(dstStorage storage.ObjectStorage) error {
- return git_model.IterateLFS(func(mo *git_model.LFSMetaObject) error {
+func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.IterateObjects(ctx, func(mo *git_model.LFSMetaObject) error {
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
return err
})
}
-func migrateAvatars(dstStorage storage.ObjectStorage) error {
- return user_model.IterateUser(func(user *user_model.User) error {
+func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.IterateObjects(ctx, func(user *user_model.User) error {
_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
return err
})
}
-func migrateRepoAvatars(dstStorage storage.ObjectStorage) error {
- return repo_model.IterateRepository(func(repo *repo_model.Repository) error {
+func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.IterateObjects(ctx, func(repo *repo_model.Repository) error {
_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
return err
})
}
+func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error {
+ p, err := archiver.RelativePath()
+ if err != nil {
+ return err
+ }
+ _, err = storage.Copy(dstStorage, p, storage.RepoArchives, p)
+ return err
+ })
+}
+
+func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.IterateObjects(ctx, func(pb *packages_model.PackageBlob) error {
+ p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
+ _, err := storage.Copy(dstStorage, p, storage.Packages, p)
+ return err
+ })
+}
+
func runMigrateStorage(ctx *cli.Context) error {
stdCtx, cancel := installSignals()
defer cancel()
@@ -127,8 +148,6 @@ func runMigrateStorage(ctx *cli.Context) error {
return err
}
- goCtx := context.Background()
-
if err := storage.Init(); err != nil {
return err
}
@@ -145,13 +164,13 @@ func runMigrateStorage(ctx *cli.Context) error {
return nil
}
dstStorage, err = storage.NewLocalStorage(
- goCtx,
+ stdCtx,
storage.LocalStorageConfig{
Path: p,
})
case string(storage.MinioStorageType):
dstStorage, err = storage.NewMinioStorage(
- goCtx,
+ stdCtx,
storage.MinioStorageConfig{
Endpoint: ctx.String("minio-endpoint"),
AccessKeyID: ctx.String("minio-access-key-id"),
@@ -162,35 +181,29 @@ func runMigrateStorage(ctx *cli.Context) error {
UseSSL: ctx.Bool("minio-use-ssl"),
})
default:
- return fmt.Errorf("Unsupported storage type: %s", ctx.String("storage"))
+ return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
}
if err != nil {
return err
}
+ migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
+ "attachments": migrateAttachments,
+ "lfs": migrateLFS,
+ "avatars": migrateAvatars,
+ "repo-avatars": migrateRepoAvatars,
+ "repo-archivers": migrateRepoArchivers,
+ "packages": migratePackages,
+ }
+
tp := strings.ToLower(ctx.String("type"))
- switch tp {
- case "attachments":
- if err := migrateAttachments(dstStorage); err != nil {
- return err
- }
- case "lfs":
- if err := migrateLFS(dstStorage); err != nil {
- return err
- }
- case "avatars":
- if err := migrateAvatars(dstStorage); err != nil {
- return err
- }
- case "repo-avatars":
- if err := migrateRepoAvatars(dstStorage); err != nil {
+ if m, ok := migratedMethods[tp]; ok {
+ if err := m(stdCtx, dstStorage); err != nil {
return err
}
- default:
- return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
+ log.Info("%s files have successfully been copied to the new storage.", tp)
+ return nil
}
- log.Warn("All files have been copied to the new placement but old files are still on the original placement.")
-
- return nil
+ return fmt.Errorf("unsupported storage: %s", ctx.String("type"))
}
diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go
new file mode 100644
index 0000000000..e6d205e410
--- /dev/null
+++ b/cmd/migrate_storage_test.go
@@ -0,0 +1,74 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+ "context"
+ "os"
+ "strings"
+ "testing"
+
+ "code.gitea.io/gitea/models/packages"
+ "code.gitea.io/gitea/models/unittest"
+ user_model "code.gitea.io/gitea/models/user"
+ packages_module "code.gitea.io/gitea/modules/packages"
+ "code.gitea.io/gitea/modules/storage"
+ packages_service "code.gitea.io/gitea/services/packages"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMigratePackages(t *testing.T) {
+ assert.NoError(t, unittest.PrepareTestDatabase())
+
+ creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
+
+ content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n"
+ buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024)
+ assert.NoError(t, err)
+ defer buf.Close()
+
+ v, f, err := packages_service.CreatePackageAndAddFile(&packages_service.PackageCreationInfo{
+ PackageInfo: packages_service.PackageInfo{
+ Owner: creator,
+ PackageType: packages.TypeGeneric,
+ Name: "test",
+ Version: "1.0.0",
+ },
+ Creator: creator,
+ SemverCompatible: true,
+ VersionProperties: map[string]string{},
+ }, &packages_service.PackageFileCreationInfo{
+ PackageFileInfo: packages_service.PackageFileInfo{
+ Filename: "a.go",
+ },
+ Data: buf,
+ IsLead: true,
+ })
+ assert.NoError(t, err)
+ assert.NotNil(t, v)
+ assert.NotNil(t, f)
+
+ ctx := context.Background()
+
+ p, err := os.MkdirTemp(os.TempDir(), "migrated_packages")
+ assert.NoError(t, err)
+
+ dstStorage, err := storage.NewLocalStorage(
+ ctx,
+ storage.LocalStorageConfig{
+ Path: p,
+ })
+ assert.NoError(t, err)
+
+ err = migratePackages(ctx, dstStorage)
+ assert.NoError(t, err)
+
+ entries, err := os.ReadDir(p)
+ assert.NoError(t, err)
+ assert.EqualValues(t, 2, len(entries))
+ assert.EqualValues(t, "01", entries[0].Name())
+ assert.EqualValues(t, "tmp", entries[1].Name())
+}
diff --git a/models/db/iterate.go b/models/db/iterate.go
new file mode 100644
index 0000000000..3d4fa06eeb
--- /dev/null
+++ b/models/db/iterate.go
@@ -0,0 +1,34 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package db
+
+import (
+ "context"
+
+ "code.gitea.io/gitea/modules/setting"
+)
+
+// IterateObjects iterate all the Bean object
+func IterateObjects[Object any](ctx context.Context, f func(repo *Object) error) error {
+ var start int
+ batchSize := setting.Database.IterateBufferSize
+ sess := GetEngine(ctx)
+ for {
+ repos := make([]*Object, 0, batchSize)
+ if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
+ return err
+ }
+ if len(repos) == 0 {
+ return nil
+ }
+ start += len(repos)
+
+ for _, repo := range repos {
+ if err := f(repo); err != nil {
+ return err
+ }
+ }
+ }
+}
diff --git a/models/git/lfs.go b/models/git/lfs.go
index ec963cf593..179da3120a 100644
--- a/models/git/lfs.go
+++ b/models/git/lfs.go
@@ -278,29 +278,6 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6
return committer.Commit()
}
-// IterateLFS iterates lfs object
-func IterateLFS(f func(mo *LFSMetaObject) error) error {
- var start int
- const batchSize = 100
- e := db.GetEngine(db.DefaultContext)
- for {
- mos := make([]*LFSMetaObject, 0, batchSize)
- if err := e.Limit(batchSize, start).Find(&mos); err != nil {
- return err
- }
- if len(mos) == 0 {
- return nil
- }
- start += len(mos)
-
- for _, mo := range mos {
- if err := f(mo); err != nil {
- return err
- }
- }
- }
-}
-
// CopyLFS copies LFS data from one repo to another
func CopyLFS(ctx context.Context, newRepo, oldRepo *repo_model.Repository) error {
var lfsObjects []*LFSMetaObject
diff --git a/models/repo/attachment.go b/models/repo/attachment.go
index ddddac2c3d..afec78a425 100644
--- a/models/repo/attachment.go
+++ b/models/repo/attachment.go
@@ -226,28 +226,6 @@ func DeleteAttachmentsByRelease(releaseID int64) error {
return err
}
-// IterateAttachment iterates attachments; it should not be used when Gitea is servicing users.
-func IterateAttachment(f func(attach *Attachment) error) error {
- var start int
- const batchSize = 100
- for {
- attachments := make([]*Attachment, 0, batchSize)
- if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&attachments); err != nil {
- return err
- }
- if len(attachments) == 0 {
- return nil
- }
- start += len(attachments)
-
- for _, attach := range attachments {
- if err := f(attach); err != nil {
- return err
- }
- }
- }
-}
-
// CountOrphanedAttachments returns the number of bad attachments
func CountOrphanedAttachments() (int64, error) {
return db.GetEngine(db.DefaultContext).Where("(issue_id > 0 and issue_id not in (select id from issue)) or (release_id > 0 and release_id not in (select id from `release`))").
diff --git a/models/repo/repo_list.go b/models/repo/repo_list.go
index 9de76fa5ff..1fa469fcfe 100644
--- a/models/repo/repo_list.go
+++ b/models/repo/repo_list.go
@@ -15,36 +15,12 @@ import (
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/container"
- "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"xorm.io/builder"
)
-// IterateRepository iterate repositories
-func IterateRepository(f func(repo *Repository) error) error {
- var start int
- batchSize := setting.Database.IterateBufferSize
- sess := db.GetEngine(db.DefaultContext)
- for {
- repos := make([]*Repository, 0, batchSize)
- if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
- return err
- }
- if len(repos) == 0 {
- return nil
- }
- start += len(repos)
-
- for _, repo := range repos {
- if err := f(repo); err != nil {
- return err
- }
- }
- }
-}
-
// FindReposMapByIDs find repos as map
func FindReposMapByIDs(repoIDs []int64, res map[int64]*Repository) error {
return db.GetEngine(db.DefaultContext).In("id", repoIDs).Find(&res)
diff --git a/models/user/search.go b/models/user/search.go
index f8e6c89f06..0aa9949367 100644
--- a/models/user/search.go
+++ b/models/user/search.go
@@ -9,7 +9,6 @@ import (
"strings"
"code.gitea.io/gitea/models/db"
- "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
@@ -125,28 +124,6 @@ func SearchUsers(opts *SearchUserOptions) (users []*User, _ int64, _ error) {
return users, count, sessQuery.Find(&users)
}
-// IterateUser iterate users
-func IterateUser(f func(user *User) error) error {
- var start int
- batchSize := setting.Database.IterateBufferSize
- for {
- users := make([]*User, 0, batchSize)
- if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&users); err != nil {
- return err
- }
- if len(users) == 0 {
- return nil
- }
- start += len(users)
-
- for _, user := range users {
- if err := f(user); err != nil {
- return err
- }
- }
- }
-}
-
// BuildCanSeeUserCondition creates a condition which can be used to restrict results to users/orgs the actor can see
func BuildCanSeeUserCondition(actor *User) builder.Cond {
if actor != nil {
diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go
index 64c3eedc23..a3a5d1a666 100644
--- a/modules/packages/content_store.go
+++ b/modules/packages/content_store.go
@@ -27,21 +27,21 @@ func NewContentStore() *ContentStore {
// Get gets a package blob
func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) {
- return s.store.Open(keyToRelativePath(key))
+ return s.store.Open(KeyToRelativePath(key))
}
// Save stores a package blob
func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error {
- _, err := s.store.Save(keyToRelativePath(key), r, size)
+ _, err := s.store.Save(KeyToRelativePath(key), r, size)
return err
}
// Delete deletes a package blob
func (s *ContentStore) Delete(key BlobHash256Key) error {
- return s.store.Delete(keyToRelativePath(key))
+ return s.store.Delete(KeyToRelativePath(key))
}
-// keyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
-func keyToRelativePath(key BlobHash256Key) string {
+// KeyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000...
+func KeyToRelativePath(key BlobHash256Key) string {
return path.Join(string(key)[0:2], string(key)[2:4], string(key))
}
diff --git a/routers/private/mail.go b/routers/private/mail.go
index 966a838168..e858992aee 100644
--- a/routers/private/mail.go
+++ b/routers/private/mail.go
@@ -9,6 +9,7 @@ import (
"net/http"
"strconv"
+ "code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/json"
@@ -59,7 +60,7 @@ func SendEmail(ctx *context.PrivateContext) {
}
}
} else {
- err := user_model.IterateUser(func(user *user_model.User) error {
+ err := db.IterateObjects(ctx, func(user *user_model.User) error {
if len(user.Email) > 0 && user.IsActive {
emails = append(emails, user.Email)
}
diff --git a/services/repository/avatar.go b/services/repository/avatar.go
index dcf04c7e54..b9bd36ab66 100644
--- a/services/repository/avatar.go
+++ b/services/repository/avatar.go
@@ -96,7 +96,7 @@ func DeleteAvatar(repo *repo_model.Repository) error {
// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
func RemoveRandomAvatars(ctx context.Context) error {
- return repo_model.IterateRepository(func(repository *repo_model.Repository) error {
+ return db.IterateObjects(ctx, func(repository *repo_model.Repository) error {
select {
case <-ctx.Done():
return db.ErrCancelledf("before random avatars removed for %s", repository.FullName())
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-27 10:31:52 +0000