aboutsummaryrefslogtreecommitdiffstats
path: root/services/mirror
diff options
context:
space:
mode:
Diffstat (limited to 'services/mirror')
-rw-r--r--services/mirror/mirror.go6
-rw-r--r--services/mirror/mirror_pull.go46
-rw-r--r--services/mirror/mirror_pull_test.go94
-rw-r--r--services/mirror/mirror_test.go66
4 files changed, 129 insertions, 83 deletions
diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go
index e029bbb1d6..7fb7fabb75 100644
--- a/services/mirror/mirror.go
+++ b/services/mirror/mirror.go
@@ -5,7 +5,7 @@ package mirror
import (
"context"
- "fmt"
+ "errors"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/log"
@@ -29,7 +29,7 @@ func doMirrorSync(ctx context.Context, req *SyncRequest) {
}
}
-var errLimit = fmt.Errorf("reached limit")
+var errLimit = errors.New("reached limit")
// Update checks and updates mirror repositories.
func Update(ctx context.Context, pullLimit, pushLimit int) error {
@@ -68,7 +68,7 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error {
// Check we've not been cancelled
select {
case <-ctx.Done():
- return fmt.Errorf("aborted")
+ return errors.New("aborted")
default:
}
diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go
index 658747e7c8..cb90af5894 100644
--- a/services/mirror/mirror_pull.go
+++ b/services/mirror/mirror_pull.go
@@ -71,7 +71,7 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error
// erase authentication before storing in database
u.User = nil
m.Repo.OriginalURL = u.String()
- return repo_model.UpdateRepositoryCols(ctx, m.Repo, "original_url")
+ return repo_model.UpdateRepositoryColsNoAutoTime(ctx, m.Repo, "original_url")
}
// mirrorSyncResult contains information of a updated reference.
@@ -235,6 +235,24 @@ func pruneBrokenReferences(ctx context.Context,
return pruneErr
}
+// checkRecoverableSyncError takes an error message from a git fetch command and returns false if it should be a fatal/blocking error
+func checkRecoverableSyncError(stderrMessage string) bool {
+ switch {
+ case strings.Contains(stderrMessage, "unable to resolve reference") && strings.Contains(stderrMessage, "reference broken"):
+ return true
+ case strings.Contains(stderrMessage, "remote error") && strings.Contains(stderrMessage, "not our ref"):
+ return true
+ case strings.Contains(stderrMessage, "cannot lock ref") && strings.Contains(stderrMessage, "but expected"):
+ return true
+ case strings.Contains(stderrMessage, "cannot lock ref") && strings.Contains(stderrMessage, "unable to resolve reference"):
+ return true
+ case strings.Contains(stderrMessage, "Unable to create") && strings.Contains(stderrMessage, ".lock"):
+ return true
+ default:
+ return false
+ }
+}
+
// runSync returns true if sync finished without error.
func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) {
repoPath := m.Repo.RepoPath()
@@ -275,7 +293,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
stdoutMessage := util.SanitizeCredentialURLs(stdout)
// Now check if the error is a resolve reference due to broken reference
- if strings.Contains(stderr, "unable to resolve reference") && strings.Contains(stderr, "reference broken") {
+ if checkRecoverableSyncError(stderr) {
log.Warn("SyncMirrors [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err)
err = nil
@@ -324,6 +342,15 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
return nil, false
}
+ if m.LFS && setting.LFS.StartServer {
+ log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
+ endpoint := lfs.DetermineEndpoint(remoteURL.String(), m.LFSEndpoint)
+ lfsClient := lfs.NewClient(endpoint, nil)
+ if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil {
+ log.Error("SyncMirrors [repo: %-v]: failed to synchronize LFS objects for repository: %v", m.Repo, err)
+ }
+ }
+
log.Trace("SyncMirrors [repo: %-v]: syncing branches...", m.Repo)
if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, m.Repo, gitRepo, 0); err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to synchronize branches: %v", m.Repo, err)
@@ -333,15 +360,6 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
if err = repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo); err != nil {
log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err)
}
-
- if m.LFS && setting.LFS.StartServer {
- log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
- endpoint := lfs.DetermineEndpoint(remoteURL.String(), m.LFSEndpoint)
- lfsClient := lfs.NewClient(endpoint, nil)
- if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil {
- log.Error("SyncMirrors [repo: %-v]: failed to synchronize LFS objects for repository: %v", m.Repo, err)
- }
- }
gitRepo.Close()
log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo)
@@ -368,7 +386,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
stdoutMessage := util.SanitizeCredentialURLs(stdout)
// Now check if the error is a resolve reference due to broken reference
- if strings.Contains(stderrMessage, "unable to resolve reference") && strings.Contains(stderrMessage, "reference broken") {
+ if checkRecoverableSyncError(stderrMessage) {
log.Warn("SyncMirrors [repo: %-v Wiki]: failed to update mirror wiki repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err)
err = nil
@@ -419,7 +437,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo
}
for _, branch := range branches {
- cache.Remove(m.Repo.GetCommitsCountCacheKey(branch.Name, true))
+ cache.Remove(m.Repo.GetCommitsCountCacheKey(branch, true))
}
m.UpdatedUnix = timeutil.TimeStampNow()
@@ -635,7 +653,7 @@ func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, re
}
m.Repo.IsEmpty = false
// Update the is empty and default_branch columns
- if err := repo_model.UpdateRepositoryCols(ctx, m.Repo, "default_branch", "is_empty"); err != nil {
+ if err := repo_model.UpdateRepositoryColsWithAutoTime(ctx, m.Repo, "default_branch", "is_empty"); err != nil {
log.Error("Failed to update default branch of repository %-v. Error: %v", m.Repo, err)
desc := fmt.Sprintf("Failed to update default branch of repository '%s': %v", m.Repo.RepoPath(), err)
if err = system_model.CreateRepositoryNotice(desc); err != nil {
diff --git a/services/mirror/mirror_pull_test.go b/services/mirror/mirror_pull_test.go
new file mode 100644
index 0000000000..97859be5b0
--- /dev/null
+++ b/services/mirror/mirror_pull_test.go
@@ -0,0 +1,94 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package mirror
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func Test_parseRemoteUpdateOutput(t *testing.T) {
+ output := `
+ * [new tag] v0.1.8 -> v0.1.8
+ * [new branch] master -> origin/master
+ - [deleted] (none) -> origin/test1
+ - [deleted] (none) -> tag1
+ + f895a1e...957a993 test2 -> origin/test2 (forced update)
+ 957a993..a87ba5f test3 -> origin/test3
+ * [new ref] refs/pull/26595/head -> refs/pull/26595/head
+ * [new ref] refs/pull/26595/merge -> refs/pull/26595/merge
+ e0639e38fb..6db2410489 refs/pull/25873/head -> refs/pull/25873/head
+ + 1c97ebc746...976d27d52f refs/pull/25873/merge -> refs/pull/25873/merge (forced update)
+`
+ results := parseRemoteUpdateOutput(output, "origin")
+ assert.Len(t, results, 10)
+ assert.Equal(t, "refs/tags/v0.1.8", results[0].refName.String())
+ assert.Equal(t, gitShortEmptySha, results[0].oldCommitID)
+ assert.Empty(t, results[0].newCommitID)
+
+ assert.Equal(t, "refs/heads/master", results[1].refName.String())
+ assert.Equal(t, gitShortEmptySha, results[1].oldCommitID)
+ assert.Empty(t, results[1].newCommitID)
+
+ assert.Equal(t, "refs/heads/test1", results[2].refName.String())
+ assert.Empty(t, results[2].oldCommitID)
+ assert.Equal(t, gitShortEmptySha, results[2].newCommitID)
+
+ assert.Equal(t, "refs/tags/tag1", results[3].refName.String())
+ assert.Empty(t, results[3].oldCommitID)
+ assert.Equal(t, gitShortEmptySha, results[3].newCommitID)
+
+ assert.Equal(t, "refs/heads/test2", results[4].refName.String())
+ assert.Equal(t, "f895a1e", results[4].oldCommitID)
+ assert.Equal(t, "957a993", results[4].newCommitID)
+
+ assert.Equal(t, "refs/heads/test3", results[5].refName.String())
+ assert.Equal(t, "957a993", results[5].oldCommitID)
+ assert.Equal(t, "a87ba5f", results[5].newCommitID)
+
+ assert.Equal(t, "refs/pull/26595/head", results[6].refName.String())
+ assert.Equal(t, gitShortEmptySha, results[6].oldCommitID)
+ assert.Empty(t, results[6].newCommitID)
+
+ assert.Equal(t, "refs/pull/26595/merge", results[7].refName.String())
+ assert.Equal(t, gitShortEmptySha, results[7].oldCommitID)
+ assert.Empty(t, results[7].newCommitID)
+
+ assert.Equal(t, "refs/pull/25873/head", results[8].refName.String())
+ assert.Equal(t, "e0639e38fb", results[8].oldCommitID)
+ assert.Equal(t, "6db2410489", results[8].newCommitID)
+
+ assert.Equal(t, "refs/pull/25873/merge", results[9].refName.String())
+ assert.Equal(t, "1c97ebc746", results[9].oldCommitID)
+ assert.Equal(t, "976d27d52f", results[9].newCommitID)
+}
+
+func Test_checkRecoverableSyncError(t *testing.T) {
+ cases := []struct {
+ recoverable bool
+ message string
+ }{
+ // A race condition in http git-fetch where certain refs were listed on the remote and are no longer there, would exit status 128
+ {true, "fatal: remote error: upload-pack: not our ref 988881adc9fc3655077dc2d4d757d480b5ea0e11"},
+ // A race condition where a local gc/prune removes a named ref during a git-fetch would exit status 1
+ {true, "cannot lock ref 'refs/pull/123456/merge': unable to resolve reference 'refs/pull/134153/merge'"},
+ // A race condition in http git-fetch where named refs were listed on the remote and are no longer there
+ {true, "error: cannot lock ref 'refs/remotes/origin/foo': unable to resolve reference 'refs/remotes/origin/foo': reference broken"},
+ // A race condition in http git-fetch where named refs were force-pushed during the update, would exit status 128
+ {true, "error: cannot lock ref 'refs/pull/123456/merge': is at 988881adc9fc3655077dc2d4d757d480b5ea0e11 but expected 7f894307ffc9553edbd0b671cab829786866f7b2"},
+ // A race condition with other local git operations, such as git-maintenance, would exit status 128 (well, "Unable" the "U" is uppercase)
+ {true, "fatal: Unable to create '/data/gitea-repositories/foo-org/bar-repo.git/./objects/info/commit-graphs/commit-graph-chain.lock': File exists."},
+ // Missing or unauthorized credentials, would exit status 128
+ {false, "fatal: Authentication failed for 'https://example.com/foo-does-not-exist/bar.git/'"},
+ // A non-existent remote repository, would exit status 128
+ {false, "fatal: Could not read from remote repository."},
+ // A non-functioning proxy, would exit status 128
+ {false, "fatal: unable to access 'https://example.com/foo-does-not-exist/bar.git/': Failed to connect to configured-https-proxy port 1080 after 0 ms: Couldn't connect to server"},
+ }
+
+ for _, c := range cases {
+ assert.Equal(t, c.recoverable, checkRecoverableSyncError(c.message), "test case: %s", c.message)
+ }
+}
diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go
deleted file mode 100644
index 76632b6872..0000000000
--- a/services/mirror/mirror_test.go
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright 2023 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package mirror
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-)
-
-func Test_parseRemoteUpdateOutput(t *testing.T) {
- output := `
- * [new tag] v0.1.8 -> v0.1.8
- * [new branch] master -> origin/master
- - [deleted] (none) -> origin/test1
- - [deleted] (none) -> tag1
- + f895a1e...957a993 test2 -> origin/test2 (forced update)
- 957a993..a87ba5f test3 -> origin/test3
- * [new ref] refs/pull/26595/head -> refs/pull/26595/head
- * [new ref] refs/pull/26595/merge -> refs/pull/26595/merge
- e0639e38fb..6db2410489 refs/pull/25873/head -> refs/pull/25873/head
- + 1c97ebc746...976d27d52f refs/pull/25873/merge -> refs/pull/25873/merge (forced update)
-`
- results := parseRemoteUpdateOutput(output, "origin")
- assert.Len(t, results, 10)
- assert.EqualValues(t, "refs/tags/v0.1.8", results[0].refName.String())
- assert.EqualValues(t, gitShortEmptySha, results[0].oldCommitID)
- assert.EqualValues(t, "", results[0].newCommitID)
-
- assert.EqualValues(t, "refs/heads/master", results[1].refName.String())
- assert.EqualValues(t, gitShortEmptySha, results[1].oldCommitID)
- assert.EqualValues(t, "", results[1].newCommitID)
-
- assert.EqualValues(t, "refs/heads/test1", results[2].refName.String())
- assert.EqualValues(t, "", results[2].oldCommitID)
- assert.EqualValues(t, gitShortEmptySha, results[2].newCommitID)
-
- assert.EqualValues(t, "refs/tags/tag1", results[3].refName.String())
- assert.EqualValues(t, "", results[3].oldCommitID)
- assert.EqualValues(t, gitShortEmptySha, results[3].newCommitID)
-
- assert.EqualValues(t, "refs/heads/test2", results[4].refName.String())
- assert.EqualValues(t, "f895a1e", results[4].oldCommitID)
- assert.EqualValues(t, "957a993", results[4].newCommitID)
-
- assert.EqualValues(t, "refs/heads/test3", results[5].refName.String())
- assert.EqualValues(t, "957a993", results[5].oldCommitID)
- assert.EqualValues(t, "a87ba5f", results[5].newCommitID)
-
- assert.EqualValues(t, "refs/pull/26595/head", results[6].refName.String())
- assert.EqualValues(t, gitShortEmptySha, results[6].oldCommitID)
- assert.EqualValues(t, "", results[6].newCommitID)
-
- assert.EqualValues(t, "refs/pull/26595/merge", results[7].refName.String())
- assert.EqualValues(t, gitShortEmptySha, results[7].oldCommitID)
- assert.EqualValues(t, "", results[7].newCommitID)
-
- assert.EqualValues(t, "refs/pull/25873/head", results[8].refName.String())
- assert.EqualValues(t, "e0639e38fb", results[8].oldCommitID)
- assert.EqualValues(t, "6db2410489", results[8].newCommitID)
-
- assert.EqualValues(t, "refs/pull/25873/merge", results[9].refName.String())
- assert.EqualValues(t, "1c97ebc746", results[9].oldCommitID)
- assert.EqualValues(t, "976d27d52f", results[9].newCommitID)
-}
generated by cgit v1.2.3 (git 2.39.1) at 2025-08-08 19:11:14 +0000