Improve sync performance for pull-mirrors (#19125)

This addresses https://github.com/go-gitea/gitea/issues/18352

It aims to improve performance (and resource use) of the `SyncReleasesWithTags` operation for pull-mirrors.

For large repositories with many tags, `SyncReleasesWithTags` can be a costly operation (taking several minutes to complete). The reason is two-fold:
    
1. on sync, every upstream repo tag is compared (for changes) against existing local entries in the release table to ensure that they are up-to-date.
    
2. the procedure for getting _each tag_ involves a series of git operations    
    ```bash
     git show-ref --tags -- v8.2.4477
     git cat-file -t 29ab6ce9f36660cffaad3c8789e71162e5db5d2f
     git cat-file -p 29ab6ce9f36660cffaad3c8789e71162e5db5d2f
     git rev-list --count 29ab6ce9f36660cffaad3c8789e71162e5db5d2f
     ```    

     of which the `git rev-list --count` can be particularly heavy.
    
This PR optimizes performance for pull-mirrors. We utilize the fact that a pull-mirror is always identical to its upstream and rebuild the entire release table on every sync and use a batch `git for-each-ref .. refs/tags` call to retrieve all tags in one go.
    
For large mirror repos, with hundreds of annotated tags, this brings down the duration of the sync operation from several minutes to a few seconds. A few unscientific examples run on my local machine:

- https://github.com/spring-projects/spring-boot (223 tags)
  - before: `0m28,673s`
  - after: `0m2,244s`
- https://github.com/kubernetes/kubernetes (890 tags)
  - before: `8m00s`
  - after: `0m8,520s`
- https://github.com/vim/vim (13954 tags)
  - before: `14m20,383s`
  - after: `0m35,467s`

 

I added a `foreachref` package which contains a flexible way of specifying which reference fields are of interest (`git-for-each-ref(1)`) and to produce a parser for the expected output. These could be reused in other places where `for-each-ref` is used.  I'll add unit tests for those if the overall PR looks promising.

1 files changed, 60 insertions, 0 deletions

diff --git a/modules/repository/repo.go b/modules/repository/repo.go
index 3ed48134c3..f1524e9efd 100644
--- a/modules/repository/repo.go
+++ b/modules/repository/repo.go
@@ -150,6 +150,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
 		}
 
 		if !opts.Releases {
+			// note: this will greatly improve release (tag) sync
+			// for pull-mirrors with many tags
+			repo.IsMirror = opts.Mirror
 			if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
 				log.Error("Failed to synchronize tags to releases for repository: %v", err)
 			}
@@ -254,6 +257,14 @@ func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo
 
 // SyncReleasesWithTags synchronizes release table with repository tags
 func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) error {
+	log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
+
+	// optimized procedure for pull-mirrors which saves a lot of time (in
+	// particular for repos with many tags).
+	if repo.IsMirror {
+		return pullMirrorReleaseSync(repo, gitRepo)
+	}
+
 	existingRelTags := make(map[string]struct{})
 	opts := models.FindReleasesOptions{
 		IncludeDrafts: true,
@@ -450,3 +461,52 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re
 
 	return nil
 }
+
+// pullMirrorReleaseSync is a pull-mirror specific tag<->release table
+// synchronization which overwrites all Releases from the repository tags. This
+// can be relied on since a pull-mirror is always identical to its
+// upstream. Hence, after each sync we want the pull-mirror release set to be
+// identical to the upstream tag set. This is much more efficient for
+// repositories like https://github.com/vim/vim (with over 13000 tags).
+func pullMirrorReleaseSync(repo *repo_model.Repository, gitRepo *git.Repository) error {
+	log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
+	tags, numTags, err := gitRepo.GetTagInfos(0, 0)
+	if err != nil {
+		return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
+	}
+	err = db.WithTx(func(ctx context.Context) error {
+		//
+		// clear out existing releases
+		//
+		if _, err := db.DeleteByBean(ctx, &models.Release{RepoID: repo.ID}); err != nil {
+			return fmt.Errorf("unable to clear releases for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
+		}
+		//
+		// make release set identical to upstream tags
+		//
+		for _, tag := range tags {
+			release := models.Release{
+				RepoID:       repo.ID,
+				TagName:      tag.Name,
+				LowerTagName: strings.ToLower(tag.Name),
+				Sha1:         tag.Object.String(),
+				// NOTE: ignored, since NumCommits are unused
+				// for pull-mirrors (only relevant when
+				// displaying releases, IsTag: false)
+				NumCommits:  -1,
+				CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()),
+				IsTag:       true,
+			}
+			if err := db.Insert(ctx, release); err != nil {
+				return fmt.Errorf("unable insert tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
+	}
+
+	log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags)
+	return nil
+}