Improve sync performance for pull-mirrors (#19125)

This addresses https://github.com/go-gitea/gitea/issues/18352

It aims to improve performance (and resource use) of the `SyncReleasesWithTags` operation for pull-mirrors.

For large repositories with many tags, `SyncReleasesWithTags` can be a costly operation (taking several minutes to complete). The reason is two-fold:
    
1. on sync, every upstream repo tag is compared (for changes) against existing local entries in the release table to ensure that they are up-to-date.
    
2. the procedure for getting _each tag_ involves a series of git operations    
    ```bash
     git show-ref --tags -- v8.2.4477
     git cat-file -t 29ab6ce9f36660cffaad3c8789e71162e5db5d2f
     git cat-file -p 29ab6ce9f36660cffaad3c8789e71162e5db5d2f
     git rev-list --count 29ab6ce9f36660cffaad3c8789e71162e5db5d2f
     ```    

     of which the `git rev-list --count` can be particularly heavy.
    
This PR optimizes performance for pull-mirrors. We utilize the fact that a pull-mirror is always identical to its upstream and rebuild the entire release table on every sync and use a batch `git for-each-ref .. refs/tags` call to retrieve all tags in one go.
    
For large mirror repos, with hundreds of annotated tags, this brings down the duration of the sync operation from several minutes to a few seconds. A few unscientific examples run on my local machine:

- https://github.com/spring-projects/spring-boot (223 tags)
  - before: `0m28,673s`
  - after: `0m2,244s`
- https://github.com/kubernetes/kubernetes (890 tags)
  - before: `8m00s`
  - after: `0m8,520s`
- https://github.com/vim/vim (13954 tags)
  - before: `14m20,383s`
  - after: `0m35,467s`

 

I added a `foreachref` package which contains a flexible way of specifying which reference fields are of interest (`git-for-each-ref(1)`) and to produce a parser for the expected output. These could be reused in other places where `for-each-ref` is used.  I'll add unit tests for those if the overall PR looks promising.

1 files changed, 84 insertions, 0 deletions

diff --git a/modules/git/foreachref/format.go b/modules/git/foreachref/format.go
new file mode 100644
index 0000000000..c9aa5233e1
--- /dev/null
+++ b/modules/git/foreachref/format.go
@@ -0,0 +1,84 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package foreachref
+
+import (
+	"encoding/hex"
+	"fmt"
+	"io"
+	"strings"
+)
+
+var (
+	nullChar     = []byte("\x00")
+	dualNullChar = []byte("\x00\x00")
+)
+
+// Format supports specifying and parsing an output format for 'git
+// for-each-ref'. See See git-for-each-ref(1) for available fields.
+type Format struct {
+	// fieldNames hold %(fieldname)s to be passed to the '--format' flag of
+	// for-each-ref. See git-for-each-ref(1) for available fields.
+	fieldNames []string
+
+	// fieldDelim is the character sequence that is used to separate fields
+	// for each reference. fieldDelim and refDelim should be selected to not
+	// interfere with each other and to not be present in field values.
+	fieldDelim []byte
+	// fieldDelimStr is a string representation of fieldDelim. Used to save
+	// us from repetitive reallocation whenever we need the delimiter as a
+	// string.
+	fieldDelimStr string
+	// refDelim is the character sequence used to separate reference from
+	// each other in the output. fieldDelim and refDelim should be selected
+	// to not interfere with each other and to not be present in field
+	// values.
+	refDelim []byte
+}
+
+// NewFormat creates a forEachRefFormat using the specified fieldNames. See
+// git-for-each-ref(1) for available fields.
+func NewFormat(fieldNames ...string) Format {
+	return Format{
+		fieldNames:    fieldNames,
+		fieldDelim:    nullChar,
+		fieldDelimStr: string(nullChar),
+		refDelim:      dualNullChar,
+	}
+}
+
+// Flag returns a for-each-ref --format flag value that captures the fieldNames.
+func (f Format) Flag() string {
+	var formatFlag strings.Builder
+	for i, field := range f.fieldNames {
+		// field key and field value
+		formatFlag.WriteString(fmt.Sprintf("%s %%(%s)", field, field))
+
+		if i < len(f.fieldNames)-1 {
+			// note: escape delimiters to allow control characters as
+			// delimiters. For example, '%00' for null character or '%0a'
+			// for newline.
+			formatFlag.WriteString(f.hexEscaped(f.fieldDelim))
+		}
+	}
+	formatFlag.WriteString(f.hexEscaped(f.refDelim))
+	return formatFlag.String()
+}
+
+// Parser returns a Parser capable of parsing 'git for-each-ref' output produced
+// with this Format.
+func (f Format) Parser(r io.Reader) *Parser {
+	return NewParser(r, f)
+}
+
+// hexEscaped produces hex-escpaed characters from a string. For example, "\n\0"
+// would turn into "%0a%00".
+func (f Format) hexEscaped(delim []byte) string {
+	escaped := ""
+	for i := 0; i < len(delim); i++ {
+		escaped += "%" + hex.EncodeToString([]byte{delim[i]})
+	}
+	return escaped
+}