Add push to remote mirror repository (#15157)

* Added push mirror model.

* Integrated push mirror into queue.

* Moved methods into own file.

* Added basic implementation.

* Mirror wiki too.

* Removed duplicated method.

* Get url for different remotes.

* Added migration.

* Unified remote url access.

* Add/Remove push mirror remotes.

* Prevent hangs with missing credentials.

* Moved code between files.

* Changed sanitizer interface.

* Added push mirror backend methods.

* Only update the mirror remote.

* Limit refs on push.

* Added UI part.

* Added missing table.

* Delete mirror if repository gets removed.

* Changed signature. Handle object errors.

* Added upload method.

* Added "upload" unit tests.

* Added transfer adapter unit tests.

* Send correct headers.

* Added pushing of LFS objects.

* Added more logging.

* Simpler body handling.

* Process files in batches to reduce HTTP calls.

* Added created timestamp.

* Fixed invalid column name.

* Changed name to prevent xorm auto setting.

* Remove table header im empty.

* Strip exit code from error message.

* Added docs page about mirroring.

* Fixed date.

* Fixed merge errors.

* Moved test to integrations.

* Added push mirror test.

* Added test.

2 files changed, 187 insertions, 35 deletions

diff --git a/modules/util/sanitize.go b/modules/util/sanitize.go
index a4f5479dfb..de59ffaa2e 100644
--- a/modules/util/sanitize.go
+++ b/modules/util/sanitize.go
@@ -1,4 +1,4 @@
-// Copyright 2017 The Gitea Authors. All rights reserved.
+// Copyright 2021 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
@@ -9,40 +9,53 @@ import (
 	"strings"
 )
 
-// urlSafeError wraps an error whose message may contain a sensitive URL
-type urlSafeError struct {
-	err            error
-	unsanitizedURL string
+const userPlaceholder = "sanitized-credential"
+const unparsableURL = "(unparsable url)"
+
+type sanitizedError struct {
+	err      error
+	replacer *strings.Replacer
 }
 
-func (err urlSafeError) Error() string {
-	return SanitizeMessage(err.err.Error(), err.unsanitizedURL)
+func (err sanitizedError) Error() string {
+	return err.replacer.Replace(err.err.Error())
 }
 
-// URLSanitizedError returns the sanitized version an error whose message may
-// contain a sensitive URL
-func URLSanitizedError(err error, unsanitizedURL string) error {
-	return urlSafeError{err: err, unsanitizedURL: unsanitizedURL}
+// NewSanitizedError wraps an error and replaces all old, new string pairs in the message text.
+func NewSanitizedError(err error, oldnew ...string) error {
+	return sanitizedError{err: err, replacer: strings.NewReplacer(oldnew...)}
 }
 
-// SanitizeMessage sanitizes a message which may contains a sensitive URL
-func SanitizeMessage(message, unsanitizedURL string) string {
-	sanitizedURL := SanitizeURLCredentials(unsanitizedURL, true)
-	return strings.ReplaceAll(message, unsanitizedURL, sanitizedURL)
+// NewURLSanitizedError wraps an error and replaces the url credential or removes them.
+func NewURLSanitizedError(err error, u *url.URL, usePlaceholder bool) error {
+	return sanitizedError{err: err, replacer: NewURLSanitizer(u, usePlaceholder)}
 }
 
-// SanitizeURLCredentials sanitizes a url, either removing user credentials
-// or replacing them with a placeholder.
-func SanitizeURLCredentials(unsanitizedURL string, usePlaceholder bool) string {
-	u, err := url.Parse(unsanitizedURL)
-	if err != nil {
-		// don't log the error, since it might contain unsanitized URL.
-		return "(unparsable url)"
-	}
+// NewStringURLSanitizedError wraps an error and replaces the url credential or removes them.
+// If the url can't get parsed it gets replaced with a placeholder string.
+func NewStringURLSanitizedError(err error, unsanitizedURL string, usePlaceholder bool) error {
+	return sanitizedError{err: err, replacer: NewStringURLSanitizer(unsanitizedURL, usePlaceholder)}
+}
+
+// NewURLSanitizer creates a replacer for the url with the credential sanitized or removed.
+func NewURLSanitizer(u *url.URL, usePlaceholder bool) *strings.Replacer {
+	old := u.String()
+
 	if u.User != nil && usePlaceholder {
-		u.User = url.User("<credentials>")
+		u.User = url.User(userPlaceholder)
 	} else {
 		u.User = nil
 	}
-	return u.String()
+	return strings.NewReplacer(old, u.String())
+}
+
+// NewStringURLSanitizer creates a replacer for the url with the credential sanitized or removed.
+// If the url can't get parsed it gets replaced with a placeholder string
+func NewStringURLSanitizer(unsanitizedURL string, usePlaceholder bool) *strings.Replacer {
+	u, err := url.Parse(unsanitizedURL)
+	if err != nil {
+		// don't log the error, since it might contain unsanitized URL.
+		return strings.NewReplacer(unsanitizedURL, unparsableURL)
+	}
+	return NewURLSanitizer(u, usePlaceholder)
 }
diff --git a/modules/util/sanitize_test.go b/modules/util/sanitize_test.go
index 4f07100675..578f75f518 100644
--- a/modules/util/sanitize_test.go
+++ b/modules/util/sanitize_test.go
@@ -1,25 +1,164 @@
-// Copyright 2020 The Gitea Authors. All rights reserved.
+// Copyright 2021 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
 package util
 
 import (
+	"errors"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestSanitizeURLCredentials(t *testing.T) {
-	var kases = map[string]string{
-		"https://github.com/go-gitea/test_repo.git":         "https://github.com/go-gitea/test_repo.git",
-		"https://mytoken@github.com/go-gitea/test_repo.git": "https://github.com/go-gitea/test_repo.git",
-		"http://github.com/go-gitea/test_repo.git":          "http://github.com/go-gitea/test_repo.git",
-		"/test/repos/repo1":                                 "/test/repos/repo1",
-		"git@github.com:go-gitea/test_repo.git":             "(unparsable url)",
+func TestNewSanitizedError(t *testing.T) {
+	err := errors.New("error while secret on test")
+	err2 := NewSanitizedError(err)
+	assert.Equal(t, err.Error(), err2.Error())
+
+	var cases = []struct {
+		input    error
+		oldnew   []string
+		expected string
+	}{
+		// case 0
+		{
+			errors.New("error while secret on test"),
+			[]string{"secret", "replaced"},
+			"error while replaced on test",
+		},
+		// case 1
+		{
+			errors.New("error while sec-ret on test"),
+			[]string{"secret", "replaced"},
+			"error while sec-ret on test",
+		},
 	}
 
-	for source, value := range kases {
-		assert.EqualValues(t, value, SanitizeURLCredentials(source, false))
+	for n, c := range cases {
+		err := NewSanitizedError(c.input, c.oldnew...)
+
+		assert.Equal(t, c.expected, err.Error(), "case %d: error should match", n)
+	}
+}
+
+func TestNewStringURLSanitizer(t *testing.T) {
+	var cases = []struct {
+		input       string
+		placeholder bool
+		expected    string
+	}{
+		// case 0
+		{
+			"https://github.com/go-gitea/test_repo.git",
+			true,
+			"https://github.com/go-gitea/test_repo.git",
+		},
+		// case 1
+		{
+			"https://github.com/go-gitea/test_repo.git",
+			false,
+			"https://github.com/go-gitea/test_repo.git",
+		},
+		// case 2
+		{
+			"https://mytoken@github.com/go-gitea/test_repo.git",
+			true,
+			"https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git",
+		},
+		// case 3
+		{
+			"https://mytoken@github.com/go-gitea/test_repo.git",
+			false,
+			"https://github.com/go-gitea/test_repo.git",
+		},
+		// case 4
+		{
+			"https://user:password@github.com/go-gitea/test_repo.git",
+			true,
+			"https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git",
+		},
+		// case 5
+		{
+			"https://user:password@github.com/go-gitea/test_repo.git",
+			false,
+			"https://github.com/go-gitea/test_repo.git",
+		},
+		// case 6
+		{
+			"https://gi\nthub.com/go-gitea/test_repo.git",
+			false,
+			unparsableURL,
+		},
+	}
+
+	for n, c := range cases {
+		// uses NewURLSanitizer internally
+		result := NewStringURLSanitizer(c.input, c.placeholder).Replace(c.input)
+
+		assert.Equal(t, c.expected, result, "case %d: error should match", n)
+	}
+}
+
+func TestNewStringURLSanitizedError(t *testing.T) {
+	var cases = []struct {
+		input       string
+		placeholder bool
+		expected    string
+	}{
+		// case 0
+		{
+			"https://github.com/go-gitea/test_repo.git",
+			true,
+			"https://github.com/go-gitea/test_repo.git",
+		},
+		// case 1
+		{
+			"https://github.com/go-gitea/test_repo.git",
+			false,
+			"https://github.com/go-gitea/test_repo.git",
+		},
+		// case 2
+		{
+			"https://mytoken@github.com/go-gitea/test_repo.git",
+			true,
+			"https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git",
+		},
+		// case 3
+		{
+			"https://mytoken@github.com/go-gitea/test_repo.git",
+			false,
+			"https://github.com/go-gitea/test_repo.git",
+		},
+		// case 4
+		{
+			"https://user:password@github.com/go-gitea/test_repo.git",
+			true,
+			"https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git",
+		},
+		// case 5
+		{
+			"https://user:password@github.com/go-gitea/test_repo.git",
+			false,
+			"https://github.com/go-gitea/test_repo.git",
+		},
+		// case 6
+		{
+			"https://gi\nthub.com/go-gitea/test_repo.git",
+			false,
+			unparsableURL,
+		},
+	}
+
+	encloseText := func(input string) string {
+		return "test " + input + " test"
+	}
+
+	for n, c := range cases {
+		err := errors.New(encloseText(c.input))
+
+		result := NewStringURLSanitizedError(err, c.input, c.placeholder)
+
+		assert.Equal(t, encloseText(c.expected), result.Error(), "case %d: error should match", n)
 	}
 }