Improve sync performance for pull-mirrors (#19125)

This addresses https://github.com/go-gitea/gitea/issues/18352

It aims to improve performance (and resource use) of the `SyncReleasesWithTags` operation for pull-mirrors.

For large repositories with many tags, `SyncReleasesWithTags` can be a costly operation (taking several minutes to complete). The reason is two-fold:
    
1. on sync, every upstream repo tag is compared (for changes) against existing local entries in the release table to ensure that they are up-to-date.
    
2. the procedure for getting _each tag_ involves a series of git operations    
    ```bash
     git show-ref --tags -- v8.2.4477
     git cat-file -t 29ab6ce9f36660cffaad3c8789e71162e5db5d2f
     git cat-file -p 29ab6ce9f36660cffaad3c8789e71162e5db5d2f
     git rev-list --count 29ab6ce9f36660cffaad3c8789e71162e5db5d2f
     ```    

     of which the `git rev-list --count` can be particularly heavy.
    
This PR optimizes performance for pull-mirrors. We utilize the fact that a pull-mirror is always identical to its upstream and rebuild the entire release table on every sync and use a batch `git for-each-ref .. refs/tags` call to retrieve all tags in one go.
    
For large mirror repos, with hundreds of annotated tags, this brings down the duration of the sync operation from several minutes to a few seconds. A few unscientific examples run on my local machine:

- https://github.com/spring-projects/spring-boot (223 tags)
  - before: `0m28,673s`
  - after: `0m2,244s`
- https://github.com/kubernetes/kubernetes (890 tags)
  - before: `8m00s`
  - after: `0m8,520s`
- https://github.com/vim/vim (13954 tags)
  - before: `14m20,383s`
  - after: `0m35,467s`

 

I added a `foreachref` package which contains a flexible way of specifying which reference fields are of interest (`git-for-each-ref(1)`) and to produce a parser for the expected output. These could be reused in other places where `for-each-ref` is used.  I'll add unit tests for those if the overall PR looks promising.

1 files changed, 131 insertions, 0 deletions

diff --git a/modules/git/foreachref/parser.go b/modules/git/foreachref/parser.go
new file mode 100644
index 0000000000..eb8b77d903
--- /dev/null
+++ b/modules/git/foreachref/parser.go
@@ -0,0 +1,131 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package foreachref
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// Parser parses 'git for-each-ref' output according to a given output Format.
+type Parser struct {
+	//  tokenizes 'git for-each-ref' output into "reference paragraphs".
+	scanner *bufio.Scanner
+
+	// format represents the '--format' string that describes the expected
+	// 'git for-each-ref' output structure.
+	format Format
+
+	// err holds the last encountered error during parsing.
+	err error
+}
+
+// NewParser creates a 'git for-each-ref' output parser that will parse all
+// references in the provided Reader. The references in the output are assumed
+// to follow the specified Format.
+func NewParser(r io.Reader, format Format) *Parser {
+	scanner := bufio.NewScanner(r)
+
+	// in addition to the reference delimiter we specified in the --format,
+	// `git for-each-ref` will always add a newline after every reference.
+	refDelim := make([]byte, 0, len(format.refDelim)+1)
+	refDelim = append(refDelim, format.refDelim...)
+	refDelim = append(refDelim, '\n')
+
+	// Split input into delimiter-separated "reference blocks".
+	scanner.Split(
+		func(data []byte, atEOF bool) (advance int, token []byte, err error) {
+			// Scan until delimiter, marking end of reference.
+			delimIdx := bytes.Index(data, refDelim)
+			if delimIdx >= 0 {
+				token := data[:delimIdx]
+				advance := delimIdx + len(refDelim)
+				return advance, token, nil
+			}
+			// If we're at EOF, we have a final, non-terminated reference. Return it.
+			if atEOF {
+				return len(data), data, nil
+			}
+			// Not yet a full field. Request more data.
+			return 0, nil, nil
+		})
+
+	return &Parser{
+		scanner: scanner,
+		format:  format,
+		err:     nil,
+	}
+}
+
+// Next returns the next reference as a collection of key-value pairs. nil
+// denotes EOF but is also returned on errors. The Err method should always be
+// consulted after Next returning nil.
+//
+// It could, for example return something like:
+//
+//  { "objecttype": "tag", "refname:short": "v1.16.4", "object": "f460b7543ed500e49c133c2cd85c8c55ee9dbe27" }
+//
+func (p *Parser) Next() map[string]string {
+	if !p.scanner.Scan() {
+		return nil
+	}
+	fields, err := p.parseRef(p.scanner.Text())
+	if err != nil {
+		p.err = err
+		return nil
+	}
+	return fields
+}
+
+// Err returns the latest encountered parsing error.
+func (p *Parser) Err() error {
+	return p.err
+}
+
+// parseRef parses out all key-value pairs from a single reference block, such as
+//
+//   "objecttype tag\0refname:short v1.16.4\0object f460b7543ed500e49c133c2cd85c8c55ee9dbe27"
+//
+func (p *Parser) parseRef(refBlock string) (map[string]string, error) {
+	if refBlock == "" {
+		// must be at EOF
+		return nil, nil
+	}
+
+	fieldValues := make(map[string]string)
+
+	fields := strings.Split(refBlock, p.format.fieldDelimStr)
+	if len(fields) != len(p.format.fieldNames) {
+		return nil, fmt.Errorf("unexpected number of reference fields: wanted %d, was %d",
+			len(fields), len(p.format.fieldNames))
+	}
+	for i, field := range fields {
+		field = strings.TrimSpace(field)
+
+		var fieldKey string
+		var fieldVal string
+		firstSpace := strings.Index(field, " ")
+		if firstSpace > 0 {
+			fieldKey = field[:firstSpace]
+			fieldVal = field[firstSpace+1:]
+		} else {
+			// could be the case if the requested field had no value
+			fieldKey = field
+		}
+
+		// enforce the format order of fields
+		if p.format.fieldNames[i] != fieldKey {
+			return nil, fmt.Errorf("unexpected field name at position %d: wanted: '%s', was: '%s'",
+				i, p.format.fieldNames[i], fieldKey)
+		}
+
+		fieldValues[fieldKey] = fieldVal
+	}
+
+	return fieldValues, nil
+}