aboutsummaryrefslogtreecommitdiffstats
path: root/services/mirror/mirror_pull.go
diff options
context:
space:
mode:
Diffstat (limited to 'services/mirror/mirror_pull.go')
-rw-r--r--services/mirror/mirror_pull.go101
1 files changed, 70 insertions, 31 deletions
diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go
index 24605cfae0..cb90af5894 100644
--- a/services/mirror/mirror_pull.go
+++ b/services/mirror/mirror_pull.go
@@ -15,6 +15,7 @@ import (
"code.gitea.io/gitea/modules/git"
giturl "code.gitea.io/gitea/modules/git/url"
"code.gitea.io/gitea/modules/gitrepo"
+ "code.gitea.io/gitea/modules/globallock"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
@@ -40,13 +41,13 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error
remoteName := m.GetRemoteName()
repoPath := m.GetRepository(ctx).RepoPath()
// Remove old remote
- _, _, err = git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(remoteName).RunStdString(&git.RunOpts{Dir: repoPath})
+ _, _, err = git.NewCommand("remote", "rm").AddDynamicArguments(remoteName).RunStdString(ctx, &git.RunOpts{Dir: repoPath})
if err != nil && !git.IsRemoteNotExistError(err) {
return err
}
- cmd := git.NewCommand(ctx, "remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(addr)
- _, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath})
+ cmd := git.NewCommand("remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(addr)
+ _, _, err = cmd.RunStdString(ctx, &git.RunOpts{Dir: repoPath})
if err != nil && !git.IsRemoteNotExistError(err) {
return err
}
@@ -55,13 +56,13 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error
wikiPath := m.Repo.WikiPath()
wikiRemotePath := repo_module.WikiRemoteURL(ctx, addr)
// Remove old remote of wiki
- _, _, err = git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(remoteName).RunStdString(&git.RunOpts{Dir: wikiPath})
+ _, _, err = git.NewCommand("remote", "rm").AddDynamicArguments(remoteName).RunStdString(ctx, &git.RunOpts{Dir: wikiPath})
if err != nil && !git.IsRemoteNotExistError(err) {
return err
}
- cmd = git.NewCommand(ctx, "remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(wikiRemotePath)
- _, _, err = cmd.RunStdString(&git.RunOpts{Dir: wikiPath})
+ cmd = git.NewCommand("remote", "add").AddDynamicArguments(remoteName).AddArguments("--mirror=fetch").AddDynamicArguments(wikiRemotePath)
+ _, _, err = cmd.RunStdString(ctx, &git.RunOpts{Dir: wikiPath})
if err != nil && !git.IsRemoteNotExistError(err) {
return err
}
@@ -70,7 +71,7 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error
// erase authentication before storing in database
u.User = nil
m.Repo.OriginalURL = u.String()
- return repo_model.UpdateRepositoryCols(ctx, m.Repo, "original_url")
+ return repo_model.UpdateRepositoryColsNoAutoTime(ctx, m.Repo, "original_url")
}
// mirrorSyncResult contains information of a updated reference.
@@ -126,7 +127,9 @@ func parseRemoteUpdateOutput(output, remoteName string) []*mirrorSyncResult {
case strings.HasPrefix(lines[i], " - "): // Delete reference
isTag := !strings.HasPrefix(refName, remoteName+"/")
var refFullName git.RefName
- if isTag {
+ if strings.HasPrefix(refName, "refs/") {
+ refFullName = git.RefName(refName)
+ } else if isTag {
refFullName = git.RefNameFromTag(refName)
} else {
refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
@@ -149,8 +152,15 @@ func parseRemoteUpdateOutput(output, remoteName string) []*mirrorSyncResult {
log.Error("Expect two SHAs but not what found: %q", lines[i])
continue
}
+ var refFullName git.RefName
+ if strings.HasPrefix(refName, "refs/") {
+ refFullName = git.RefName(refName)
+ } else {
+ refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
+ }
+
results = append(results, &mirrorSyncResult{
- refName: git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/")),
+ refName: refFullName,
oldCommitID: shas[0],
newCommitID: shas[1],
})
@@ -199,8 +209,8 @@ func pruneBrokenReferences(ctx context.Context,
stderrBuilder.Reset()
stdoutBuilder.Reset()
- pruneErr := git.NewCommand(ctx, "remote", "prune").AddDynamicArguments(m.GetRemoteName()).
- Run(&git.RunOpts{
+ pruneErr := git.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).
+ Run(ctx, &git.RunOpts{
Timeout: timeout,
Dir: repoPath,
Stdout: stdoutBuilder,
@@ -225,6 +235,24 @@ func pruneBrokenReferences(ctx context.Context,
return pruneErr
}
+// checkRecoverableSyncError takes an error message from a git fetch command and returns false if it should be a fatal/blocking error
+func checkRecoverableSyncError(stderrMessage string) bool {
+ switch {
+ case strings.Contains(stderrMessage, "unable to resolve reference") && strings.Contains(stderrMessage, "reference broken"):
+ return true
+ case strings.Contains(stderrMessage, "remote error") && strings.Contains(stderrMessage, "not our ref"):
+ return true
+ case strings.Contains(stderrMessage, "cannot lock ref") && strings.Contains(stderrMessage, "but expected"):
+ return true
+ case strings.Contains(stderrMessage, "cannot lock ref") && strings.Contains(stderrMessage, "unable to resolve reference"):
+ return true
+ case strings.Contains(stderrMessage, "Unable to create") && strings.Contains(stderrMessage, ".lock"):
+ return true
+ default:
+ return false
+ }
+}
+
// runSync returns true if sync finished without error.
func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) {
repoPath := m.Repo.RepoPath()
@@ -234,7 +262,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo)
// use fetch but not remote update because git fetch support --tags but remote update doesn't
- cmd := git.NewCommand(ctx, "fetch")
+ cmd := git.NewCommand("fetch")
if m.EnablePrune {
cmd.AddArguments("--prune")
}
@@ -250,7 +278,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
stdoutBuilder := strings.Builder{}
stderrBuilder := strings.Builder{}
- if err := cmd.Run(&git.RunOpts{
+ if err := cmd.Run(ctx, &git.RunOpts{
Timeout: timeout,
Dir: repoPath,
Env: envs,
@@ -265,7 +293,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
stdoutMessage := util.SanitizeCredentialURLs(stdout)
// Now check if the error is a resolve reference due to broken reference
- if strings.Contains(stderr, "unable to resolve reference") && strings.Contains(stderr, "reference broken") {
+ if checkRecoverableSyncError(stderr) {
log.Warn("SyncMirrors [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err)
err = nil
@@ -275,7 +303,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
// Successful prune - reattempt mirror
stderrBuilder.Reset()
stdoutBuilder.Reset()
- if err = cmd.Run(&git.RunOpts{
+ if err = cmd.Run(ctx, &git.RunOpts{
Timeout: timeout,
Dir: repoPath,
Stdout: &stdoutBuilder,
@@ -314,6 +342,15 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
return nil, false
}
+ if m.LFS && setting.LFS.StartServer {
+ log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
+ endpoint := lfs.DetermineEndpoint(remoteURL.String(), m.LFSEndpoint)
+ lfsClient := lfs.NewClient(endpoint, nil)
+ if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil {
+ log.Error("SyncMirrors [repo: %-v]: failed to synchronize LFS objects for repository: %v", m.Repo, err)
+ }
+ }
+
log.Trace("SyncMirrors [repo: %-v]: syncing branches...", m.Repo)
if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, m.Repo, gitRepo, 0); err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to synchronize branches: %v", m.Repo, err)
@@ -323,15 +360,6 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
if err = repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo); err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err)
}
-
- if m.LFS && setting.LFS.StartServer {
- log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
- endpoint := lfs.DetermineEndpoint(remoteURL.String(), m.LFSEndpoint)
- lfsClient := lfs.NewClient(endpoint, nil)
- if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil {
- log.Error("SyncMirrors [repo: %-v]: failed to synchronize LFS objects for repository: %v", m.Repo, err)
- }
- }
gitRepo.Close()
log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo)
@@ -343,8 +371,8 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo)
stderrBuilder.Reset()
stdoutBuilder.Reset()
- if err := git.NewCommand(ctx, "remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).
- Run(&git.RunOpts{
+ if err := git.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).
+ Run(ctx, &git.RunOpts{
Timeout: timeout,
Dir: wikiPath,
Stdout: &stdoutBuilder,
@@ -358,7 +386,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
stdoutMessage := util.SanitizeCredentialURLs(stdout)
// Now check if the error is a resolve reference due to broken reference
- if strings.Contains(stderrMessage, "unable to resolve reference") && strings.Contains(stderrMessage, "reference broken") {
+ if checkRecoverableSyncError(stderrMessage) {
log.Warn("SyncMirrors [repo: %-v Wiki]: failed to update mirror wiki repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err)
err = nil
@@ -369,8 +397,8 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
stderrBuilder.Reset()
stdoutBuilder.Reset()
- if err = git.NewCommand(ctx, "remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).
- Run(&git.RunOpts{
+ if err = git.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).
+ Run(ctx, &git.RunOpts{
Timeout: timeout,
Dir: wikiPath,
Stdout: &stdoutBuilder,
@@ -409,13 +437,17 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
}
for _, branch := range branches {
- cache.Remove(m.Repo.GetCommitsCountCacheKey(branch.Name, true))
+ cache.Remove(m.Repo.GetCommitsCountCacheKey(branch, true))
}
m.UpdatedUnix = timeutil.TimeStampNow()
return parseRemoteUpdateOutput(output, m.GetRemoteName()), true
}
+func getRepoPullMirrorLockKey(repoID int64) string {
+ return fmt.Sprintf("repo_pull_mirror_%d", repoID)
+}
+
// SyncPullMirror starts the sync of the pull mirror and schedules the next run.
func SyncPullMirror(ctx context.Context, repoID int64) bool {
log.Trace("SyncMirrors [repo_id: %v]", repoID)
@@ -428,6 +460,13 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
log.Error("PANIC whilst SyncMirrors[repo_id: %d] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2))
}()
+ releaser, err := globallock.Lock(ctx, getRepoPullMirrorLockKey(repoID))
+ if err != nil {
+ log.Error("globallock.Lock(): %v", err)
+ return false
+ }
+ defer releaser()
+
m, err := repo_model.GetMirrorByRepoID(ctx, repoID)
if err != nil {
log.Error("SyncMirrors [repo_id: %v]: unable to GetMirrorByRepoID: %v", repoID, err)
@@ -614,7 +653,7 @@ func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, re
}
m.Repo.IsEmpty = false
// Update the is empty and default_branch columns
- if err := repo_model.UpdateRepositoryCols(ctx, m.Repo, "default_branch", "is_empty"); err != nil {
+ if err := repo_model.UpdateRepositoryColsWithAutoTime(ctx, m.Repo, "default_branch", "is_empty"); err != nil {
log.Error("Failed to update default branch of repository %-v. Error: %v", m.Repo, err)
desc := fmt.Sprintf("Failed to update default branch of repository '%s': %v", m.Repo.RepoPath(), err)
if err = system_model.CreateRepositoryNotice(desc); err != nil {
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 05:14:17 +0000