diff options
author | Lunny Xiao <xiaolunwen@gmail.com> | 2024-03-01 10:23:00 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-03-01 02:23:00 +0000 |
commit | 58ce1de994c2a036ebf7137c143ce193694d740d (patch) | |
tree | 0e88485e095f31a9124e2643d528d323c30b9ce5 | |
parent | 5e32cd6beb1a4f11bd19e6d44ba2a50828b686ef (diff) | |
download | gitea-58ce1de994c2a036ebf7137c143ce193694d740d.tar.gz gitea-58ce1de994c2a036ebf7137c143ce193694d740d.zip |
Move migration functions to services layer (#29497)
-rw-r--r-- | modules/repository/repo.go | 243 | ||||
-rw-r--r-- | routers/web/repo/setting/setting.go | 3 | ||||
-rw-r--r-- | services/migrations/gitea_uploader.go | 2 | ||||
-rw-r--r-- | services/repository/migrate.go | 264 | ||||
-rw-r--r-- | tests/integration/mirror_pull_test.go | 3 |
5 files changed, 267 insertions, 248 deletions
diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 39bdc6adcf..cb926084ba 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -5,16 +5,13 @@ package repository import ( "context" - "errors" "fmt" "io" - "net/http" "strings" "time" "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" - "code.gitea.io/gitea/models/organization" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" @@ -22,10 +19,8 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" ) /* @@ -47,244 +42,6 @@ func WikiRemoteURL(ctx context.Context, remote string) string { return "" } -// MigrateRepositoryGitData starts migrating git related data after created migrating repository -func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, - repo *repo_model.Repository, opts migration.MigrateOptions, - httpTransport *http.Transport, -) (*repo_model.Repository, error) { - repoPath := repo_model.RepoPath(u.Name, opts.RepoName) - - if u.IsOrganization() { - t, err := organization.OrgFromUser(u).GetOwnerTeam(ctx) - if err != nil { - return nil, err - } - repo.NumWatches = t.NumMembers - } else { - repo.NumWatches = 1 - } - - migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second - - var err error - if err = util.RemoveAll(repoPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", repoPath, err) - } - - if err = git.Clone(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{ - Mirror: true, - Quiet: true, - Timeout: migrateTimeout, - SkipTLSVerify: setting.Migrations.SkipTLSVerify, - }); err != nil { - if errors.Is(err, context.DeadlineExceeded) { - return repo, fmt.Errorf("Clone timed out. Consider increasing [git.timeout] MIGRATE in app.ini. Underlying Error: %w", err) - } - return repo, fmt.Errorf("Clone: %w", err) - } - - if err := git.WriteCommitGraph(ctx, repoPath); err != nil { - return repo, err - } - - if opts.Wiki { - wikiPath := repo_model.WikiPath(u.Name, opts.RepoName) - wikiRemotePath := WikiRemoteURL(ctx, opts.CloneAddr) - if len(wikiRemotePath) > 0 { - if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) - } - - if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ - Mirror: true, - Quiet: true, - Timeout: migrateTimeout, - Branch: "master", - SkipTLSVerify: setting.Migrations.SkipTLSVerify, - }); err != nil { - log.Warn("Clone wiki: %v", err) - if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) - } - } else { - if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { - return repo, err - } - } - } - } - - if repo.OwnerID == u.ID { - repo.Owner = u - } - - if err = CheckDaemonExportOK(ctx, repo); err != nil { - return repo, fmt.Errorf("checkDaemonExportOK: %w", err) - } - - if stdout, _, err := git.NewCommand(ctx, "update-server-info"). - SetDescription(fmt.Sprintf("MigrateRepositoryGitData(git update-server-info): %s", repoPath)). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("MigrateRepositoryGitData(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - return repo, fmt.Errorf("error in MigrateRepositoryGitData(git update-server-info): %w", err) - } - - gitRepo, err := git.OpenRepository(ctx, repoPath) - if err != nil { - return repo, fmt.Errorf("OpenRepository: %w", err) - } - defer gitRepo.Close() - - repo.IsEmpty, err = gitRepo.IsEmpty() - if err != nil { - return repo, fmt.Errorf("git.IsEmpty: %w", err) - } - - if !repo.IsEmpty { - if len(repo.DefaultBranch) == 0 { - // Try to get HEAD branch and set it as default branch. - headBranch, err := gitRepo.GetHEADBranch() - if err != nil { - return repo, fmt.Errorf("GetHEADBranch: %w", err) - } - if headBranch != nil { - repo.DefaultBranch = headBranch.Name - } - } - - if _, err := SyncRepoBranchesWithRepo(ctx, repo, gitRepo, u.ID); err != nil { - return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err) - } - - if !opts.Releases { - // note: this will greatly improve release (tag) sync - // for pull-mirrors with many tags - repo.IsMirror = opts.Mirror - if err = SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { - log.Error("Failed to synchronize tags to releases for repository: %v", err) - } - } - - if opts.LFS { - endpoint := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) - lfsClient := lfs.NewClient(endpoint, httpTransport) - if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil { - log.Error("Failed to store missing LFS objects for repository: %v", err) - } - } - } - - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return nil, err - } - defer committer.Close() - - if opts.Mirror { - remoteAddress, err := util.SanitizeURL(opts.CloneAddr) - if err != nil { - return repo, err - } - mirrorModel := repo_model.Mirror{ - RepoID: repo.ID, - Interval: setting.Mirror.DefaultInterval, - EnablePrune: true, - NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), - LFS: opts.LFS, - RemoteAddress: remoteAddress, - } - if opts.LFS { - mirrorModel.LFSEndpoint = opts.LFSEndpoint - } - - if opts.MirrorInterval != "" { - parsedInterval, err := time.ParseDuration(opts.MirrorInterval) - if err != nil { - log.Error("Failed to set Interval: %v", err) - return repo, err - } - if parsedInterval == 0 { - mirrorModel.Interval = 0 - mirrorModel.NextUpdateUnix = 0 - } else if parsedInterval < setting.Mirror.MinInterval { - err := fmt.Errorf("interval %s is set below Minimum Interval of %s", parsedInterval, setting.Mirror.MinInterval) - log.Error("Interval: %s is too frequent", opts.MirrorInterval) - return repo, err - } else { - mirrorModel.Interval = parsedInterval - mirrorModel.NextUpdateUnix = timeutil.TimeStampNow().AddDuration(parsedInterval) - } - } - - if err = repo_model.InsertMirror(ctx, &mirrorModel); err != nil { - return repo, fmt.Errorf("InsertOne: %w", err) - } - - repo.IsMirror = true - if err = UpdateRepository(ctx, repo, false); err != nil { - return nil, err - } - - // this is necessary for sync local tags from remote - configName := fmt.Sprintf("remote.%s.fetch", mirrorModel.GetRemoteName()) - if stdout, _, err := git.NewCommand(ctx, "config"). - AddOptionValues("--add", configName, `+refs/tags/*:refs/tags/*`). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("MigrateRepositoryGitData(git config --add <remote> +refs/tags/*:refs/tags/*) in %v: Stdout: %s\nError: %v", repo, stdout, err) - return repo, fmt.Errorf("error in MigrateRepositoryGitData(git config --add <remote> +refs/tags/*:refs/tags/*): %w", err) - } - } else { - if err = UpdateRepoSize(ctx, repo); err != nil { - log.Error("Failed to update size for repository: %v", err) - } - if repo, err = CleanUpMigrateInfo(ctx, repo); err != nil { - return nil, err - } - } - - return repo, committer.Commit() -} - -// cleanUpMigrateGitConfig removes mirror info which prevents "push --all". -// This also removes possible user credentials. -func cleanUpMigrateGitConfig(ctx context.Context, repoPath string) error { - cmd := git.NewCommand(ctx, "remote", "rm", "origin") - // if the origin does not exist - _, stderr, err := cmd.RunStdString(&git.RunOpts{ - Dir: repoPath, - }) - if err != nil && !strings.HasPrefix(stderr, "fatal: No such remote") { - return err - } - return nil -} - -// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. -func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo_model.Repository, error) { - repoPath := repo.RepoPath() - if err := CreateDelegateHooks(repoPath); err != nil { - return repo, fmt.Errorf("createDelegateHooks: %w", err) - } - if repo.HasWiki() { - if err := CreateDelegateHooks(repo.WikiPath()); err != nil { - return repo, fmt.Errorf("createDelegateHooks.(wiki): %w", err) - } - } - - _, _, err := git.NewCommand(ctx, "remote", "rm", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return repo, fmt.Errorf("CleanUpMigrateInfo: %w", err) - } - - if repo.HasWiki() { - if err := cleanUpMigrateGitConfig(ctx, repo.WikiPath()); err != nil { - return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %w", err) - } - } - - return repo, UpdateRepository(ctx, repo, false) -} - // SyncRepoTags synchronizes releases table with repository tags func SyncRepoTags(ctx context.Context, repoID int64) error { repo, err := repo_model.GetRepositoryByID(ctx, repoID) diff --git a/routers/web/repo/setting/setting.go b/routers/web/repo/setting/setting.go index b13c4c2ddb..0f649acba3 100644 --- a/routers/web/repo/setting/setting.go +++ b/routers/web/repo/setting/setting.go @@ -23,7 +23,6 @@ import ( "code.gitea.io/gitea/modules/indexer/stats" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" - repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -693,7 +692,7 @@ func SettingsPost(ctx *context.Context) { } repo.IsMirror = false - if _, err := repo_module.CleanUpMigrateInfo(ctx, repo); err != nil { + if _, err := repo_service.CleanUpMigrateInfo(ctx, repo); err != nil { ctx.ServerError("CleanUpMigrateInfo", err) return } else if err = repo_model.DeleteMirrorByRepoID(ctx, ctx.Repo.Repository.ID); err != nil { diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 1c9824fe3a..87691bf729 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -120,7 +120,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate r.DefaultBranch = repo.DefaultBranch r.Description = repo.Description - r, err = repo_module.MigrateRepositoryGitData(g.ctx, owner, r, base.MigrateOptions{ + r, err = repo_service.MigrateRepositoryGitData(g.ctx, owner, r, base.MigrateOptions{ RepoName: g.repoName, Description: repo.Description, OriginalURL: repo.OriginalURL, diff --git a/services/repository/migrate.go b/services/repository/migrate.go new file mode 100644 index 0000000000..51fdd90f54 --- /dev/null +++ b/services/repository/migrate.go @@ -0,0 +1,264 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/organization" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migration" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) + +// MigrateRepositoryGitData starts migrating git related data after created migrating repository +func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, + repo *repo_model.Repository, opts migration.MigrateOptions, + httpTransport *http.Transport, +) (*repo_model.Repository, error) { + repoPath := repo_model.RepoPath(u.Name, opts.RepoName) + + if u.IsOrganization() { + t, err := organization.OrgFromUser(u).GetOwnerTeam(ctx) + if err != nil { + return nil, err + } + repo.NumWatches = t.NumMembers + } else { + repo.NumWatches = 1 + } + + migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second + + var err error + if err = util.RemoveAll(repoPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", repoPath, err) + } + + if err = git.Clone(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + SkipTLSVerify: setting.Migrations.SkipTLSVerify, + }); err != nil { + if errors.Is(err, context.DeadlineExceeded) { + return repo, fmt.Errorf("Clone timed out. Consider increasing [git.timeout] MIGRATE in app.ini. Underlying Error: %w", err) + } + return repo, fmt.Errorf("Clone: %w", err) + } + + if err := git.WriteCommitGraph(ctx, repoPath); err != nil { + return repo, err + } + + if opts.Wiki { + wikiPath := repo_model.WikiPath(u.Name, opts.RepoName) + wikiRemotePath := repo_module.WikiRemoteURL(ctx, opts.CloneAddr) + if len(wikiRemotePath) > 0 { + if err := util.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + } + + if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + Branch: "master", + SkipTLSVerify: setting.Migrations.SkipTLSVerify, + }); err != nil { + log.Warn("Clone wiki: %v", err) + if err := util.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + } + } else { + if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { + return repo, err + } + } + } + } + + if repo.OwnerID == u.ID { + repo.Owner = u + } + + if err = repo_module.CheckDaemonExportOK(ctx, repo); err != nil { + return repo, fmt.Errorf("checkDaemonExportOK: %w", err) + } + + if stdout, _, err := git.NewCommand(ctx, "update-server-info"). + SetDescription(fmt.Sprintf("MigrateRepositoryGitData(git update-server-info): %s", repoPath)). + RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + log.Error("MigrateRepositoryGitData(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return repo, fmt.Errorf("error in MigrateRepositoryGitData(git update-server-info): %w", err) + } + + gitRepo, err := git.OpenRepository(ctx, repoPath) + if err != nil { + return repo, fmt.Errorf("OpenRepository: %w", err) + } + defer gitRepo.Close() + + repo.IsEmpty, err = gitRepo.IsEmpty() + if err != nil { + return repo, fmt.Errorf("git.IsEmpty: %w", err) + } + + if !repo.IsEmpty { + if len(repo.DefaultBranch) == 0 { + // Try to get HEAD branch and set it as default branch. + headBranch, err := gitRepo.GetHEADBranch() + if err != nil { + return repo, fmt.Errorf("GetHEADBranch: %w", err) + } + if headBranch != nil { + repo.DefaultBranch = headBranch.Name + } + } + + if _, err := repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, u.ID); err != nil { + return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err) + } + + if !opts.Releases { + // note: this will greatly improve release (tag) sync + // for pull-mirrors with many tags + repo.IsMirror = opts.Mirror + if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { + log.Error("Failed to synchronize tags to releases for repository: %v", err) + } + } + + if opts.LFS { + endpoint := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) + lfsClient := lfs.NewClient(endpoint, httpTransport) + if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil { + log.Error("Failed to store missing LFS objects for repository: %v", err) + } + } + } + + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return nil, err + } + defer committer.Close() + + if opts.Mirror { + remoteAddress, err := util.SanitizeURL(opts.CloneAddr) + if err != nil { + return repo, err + } + mirrorModel := repo_model.Mirror{ + RepoID: repo.ID, + Interval: setting.Mirror.DefaultInterval, + EnablePrune: true, + NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), + LFS: opts.LFS, + RemoteAddress: remoteAddress, + } + if opts.LFS { + mirrorModel.LFSEndpoint = opts.LFSEndpoint + } + + if opts.MirrorInterval != "" { + parsedInterval, err := time.ParseDuration(opts.MirrorInterval) + if err != nil { + log.Error("Failed to set Interval: %v", err) + return repo, err + } + if parsedInterval == 0 { + mirrorModel.Interval = 0 + mirrorModel.NextUpdateUnix = 0 + } else if parsedInterval < setting.Mirror.MinInterval { + err := fmt.Errorf("interval %s is set below Minimum Interval of %s", parsedInterval, setting.Mirror.MinInterval) + log.Error("Interval: %s is too frequent", opts.MirrorInterval) + return repo, err + } else { + mirrorModel.Interval = parsedInterval + mirrorModel.NextUpdateUnix = timeutil.TimeStampNow().AddDuration(parsedInterval) + } + } + + if err = repo_model.InsertMirror(ctx, &mirrorModel); err != nil { + return repo, fmt.Errorf("InsertOne: %w", err) + } + + repo.IsMirror = true + if err = UpdateRepository(ctx, repo, false); err != nil { + return nil, err + } + + // this is necessary for sync local tags from remote + configName := fmt.Sprintf("remote.%s.fetch", mirrorModel.GetRemoteName()) + if stdout, _, err := git.NewCommand(ctx, "config"). + AddOptionValues("--add", configName, `+refs/tags/*:refs/tags/*`). + RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + log.Error("MigrateRepositoryGitData(git config --add <remote> +refs/tags/*:refs/tags/*) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return repo, fmt.Errorf("error in MigrateRepositoryGitData(git config --add <remote> +refs/tags/*:refs/tags/*): %w", err) + } + } else { + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + if repo, err = CleanUpMigrateInfo(ctx, repo); err != nil { + return nil, err + } + } + + return repo, committer.Commit() +} + +// cleanUpMigrateGitConfig removes mirror info which prevents "push --all". +// This also removes possible user credentials. +func cleanUpMigrateGitConfig(ctx context.Context, repoPath string) error { + cmd := git.NewCommand(ctx, "remote", "rm", "origin") + // if the origin does not exist + _, stderr, err := cmd.RunStdString(&git.RunOpts{ + Dir: repoPath, + }) + if err != nil && !strings.HasPrefix(stderr, "fatal: No such remote") { + return err + } + return nil +} + +// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. +func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo_model.Repository, error) { + repoPath := repo.RepoPath() + if err := repo_module.CreateDelegateHooks(repoPath); err != nil { + return repo, fmt.Errorf("createDelegateHooks: %w", err) + } + if repo.HasWiki() { + if err := repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil { + return repo, fmt.Errorf("createDelegateHooks.(wiki): %w", err) + } + } + + _, _, err := git.NewCommand(ctx, "remote", "rm", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return repo, fmt.Errorf("CleanUpMigrateInfo: %w", err) + } + + if repo.HasWiki() { + if err := cleanUpMigrateGitConfig(ctx, repo.WikiPath()); err != nil { + return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %w", err) + } + } + + return repo, UpdateRepository(ctx, repo, false) +} diff --git a/tests/integration/mirror_pull_test.go b/tests/integration/mirror_pull_test.go index 2e71b80fbb..77050c4bbc 100644 --- a/tests/integration/mirror_pull_test.go +++ b/tests/integration/mirror_pull_test.go @@ -14,7 +14,6 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/migration" - "code.gitea.io/gitea/modules/repository" mirror_service "code.gitea.io/gitea/services/mirror" release_service "code.gitea.io/gitea/services/release" repo_service "code.gitea.io/gitea/services/repository" @@ -52,7 +51,7 @@ func TestMirrorPull(t *testing.T) { ctx := context.Background() - mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts, nil) + mirror, err := repo_service.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts, nil) assert.NoError(t, err) gitRepo, err := gitrepo.OpenRepository(git.DefaultContext, repo) |