Add push to remote mirror repository (#15157)

* Added push mirror model.

* Integrated push mirror into queue.

* Moved methods into own file.

* Added basic implementation.

* Mirror wiki too.

* Removed duplicated method.

* Get url for different remotes.

* Added migration.

* Unified remote url access.

* Add/Remove push mirror remotes.

* Prevent hangs with missing credentials.

* Moved code between files.

* Changed sanitizer interface.

* Added push mirror backend methods.

* Only update the mirror remote.

* Limit refs on push.

* Added UI part.

* Added missing table.

* Delete mirror if repository gets removed.

* Changed signature. Handle object errors.

* Added upload method.

* Added "upload" unit tests.

* Added transfer adapter unit tests.

* Send correct headers.

* Added pushing of LFS objects.

* Added more logging.

* Simpler body handling.

* Process files in batches to reduce HTTP calls.

* Added created timestamp.

* Fixed invalid column name.

* Changed name to prevent xorm auto setting.

* Remove table header im empty.

* Strip exit code from error message.

* Added docs page about mirroring.

* Fixed date.

* Fixed merge errors.

* Moved test to integrations.

* Added push mirror test.

* Added test.

1 files changed, 268 insertions, 52 deletions

diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go
index 68ec947aa8..0f633ede54 100644
--- a/modules/lfs/http_client_test.go
+++ b/modules/lfs/http_client_test.go
@@ -7,13 +7,13 @@ package lfs
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"strings"
 	"testing"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -30,69 +30,253 @@ func (a *DummyTransferAdapter) Name() string {
 	return "dummy"
 }
 
-func (a *DummyTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) {
+func (a *DummyTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) {
 	return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil
 }
 
-func TestHTTPClientDownload(t *testing.T) {
-	oid := "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041"
-	size := int64(6)
+func (a *DummyTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error {
+	return nil
+}
+
+func (a *DummyTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error {
+	return nil
+}
+
+func lfsTestRoundtripHandler(req *http.Request) *http.Response {
+	var batchResponse *BatchResponse
+	url := req.URL.String()
 
-	roundTripHandler := func(req *http.Request) *http.Response {
-		url := req.URL.String()
-		if strings.Contains(url, "status-not-ok") {
-			return &http.Response{StatusCode: http.StatusBadRequest}
+	if strings.Contains(url, "status-not-ok") {
+		return &http.Response{StatusCode: http.StatusBadRequest}
+	} else if strings.Contains(url, "invalid-json-response") {
+		return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))}
+	} else if strings.Contains(url, "valid-batch-request-download") {
+		batchResponse = &BatchResponse{
+			Transfer: "dummy",
+			Objects: []*ObjectResponse{
+				{
+					Actions: map[string]*Link{
+						"download": {},
+					},
+				},
+			},
+		}
+	} else if strings.Contains(url, "valid-batch-request-upload") {
+		batchResponse = &BatchResponse{
+			Transfer: "dummy",
+			Objects: []*ObjectResponse{
+				{
+					Actions: map[string]*Link{
+						"upload": {},
+					},
+				},
+			},
 		}
-		if strings.Contains(url, "invalid-json-response") {
-			return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))}
+	} else if strings.Contains(url, "response-no-objects") {
+		batchResponse = &BatchResponse{Transfer: "dummy"}
+	} else if strings.Contains(url, "unknown-transfer-adapter") {
+		batchResponse = &BatchResponse{Transfer: "unknown_adapter"}
+	} else if strings.Contains(url, "error-in-response-objects") {
+		batchResponse = &BatchResponse{
+			Transfer: "dummy",
+			Objects: []*ObjectResponse{
+				{
+					Error: &ObjectError{
+						Code:    404,
+						Message: "Object not found",
+					},
+				},
+			},
 		}
-		if strings.Contains(url, "valid-batch-request-download") {
-			assert.Equal(t, "POST", req.Method)
-			assert.Equal(t, MediaType, req.Header.Get("Content-type"), "case %s: error should match", url)
-			assert.Equal(t, MediaType, req.Header.Get("Accept"), "case %s: error should match", url)
+	} else if strings.Contains(url, "empty-actions-map") {
+		batchResponse = &BatchResponse{
+			Transfer: "dummy",
+			Objects: []*ObjectResponse{
+				{
+					Actions: map[string]*Link{},
+				},
+			},
+		}
+	} else if strings.Contains(url, "download-actions-map") {
+		batchResponse = &BatchResponse{
+			Transfer: "dummy",
+			Objects: []*ObjectResponse{
+				{
+					Actions: map[string]*Link{
+						"download": {},
+					},
+				},
+			},
+		}
+	} else if strings.Contains(url, "upload-actions-map") {
+		batchResponse = &BatchResponse{
+			Transfer: "dummy",
+			Objects: []*ObjectResponse{
+				{
+					Actions: map[string]*Link{
+						"upload": {},
+					},
+				},
+			},
+		}
+	} else if strings.Contains(url, "verify-actions-map") {
+		batchResponse = &BatchResponse{
+			Transfer: "dummy",
+			Objects: []*ObjectResponse{
+				{
+					Actions: map[string]*Link{
+						"verify": {},
+					},
+				},
+			},
+		}
+	} else if strings.Contains(url, "unknown-actions-map") {
+		batchResponse = &BatchResponse{
+			Transfer: "dummy",
+			Objects: []*ObjectResponse{
+				{
+					Actions: map[string]*Link{
+						"unknown": {},
+					},
+				},
+			},
+		}
+	} else {
+		return nil
+	}
 
-			var batchRequest BatchRequest
-			err := json.NewDecoder(req.Body).Decode(&batchRequest)
-			assert.NoError(t, err)
+	payload := new(bytes.Buffer)
+	jsoniter.NewEncoder(payload).Encode(batchResponse)
 
-			assert.Equal(t, "download", batchRequest.Operation)
-			assert.Len(t, batchRequest.Objects, 1)
-			assert.Equal(t, oid, batchRequest.Objects[0].Oid)
-			assert.Equal(t, size, batchRequest.Objects[0].Size)
+	return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
+}
 
-			batchResponse := &BatchResponse{
-				Transfer: "dummy",
-				Objects:  make([]*ObjectResponse, 1),
-			}
+func TestHTTPClientDownload(t *testing.T) {
+	p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6}
 
-			payload := new(bytes.Buffer)
-			json.NewEncoder(payload).Encode(batchResponse)
+	hc := &http.Client{Transport: RoundTripFunc(func(req *http.Request) *http.Response {
+		assert.Equal(t, "POST", req.Method)
+		assert.Equal(t, MediaType, req.Header.Get("Content-type"))
+		assert.Equal(t, MediaType, req.Header.Get("Accept"))
 
-			return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
-		}
-		if strings.Contains(url, "invalid-response-no-objects") {
-			batchResponse := &BatchResponse{Transfer: "dummy"}
+		var batchRequest BatchRequest
+		err := jsoniter.NewDecoder(req.Body).Decode(&batchRequest)
+		assert.NoError(t, err)
 
-			payload := new(bytes.Buffer)
-			json.NewEncoder(payload).Encode(batchResponse)
+		assert.Equal(t, "download", batchRequest.Operation)
+		assert.Equal(t, 1, len(batchRequest.Objects))
+		assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid)
+		assert.Equal(t, p.Size, batchRequest.Objects[0].Size)
 
-			return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
-		}
-		if strings.Contains(url, "unknown-transfer-adapter") {
-			batchResponse := &BatchResponse{Transfer: "unknown_adapter"}
+		return lfsTestRoundtripHandler(req)
+	})}
+	dummy := &DummyTransferAdapter{}
 
-			payload := new(bytes.Buffer)
-			json.NewEncoder(payload).Encode(batchResponse)
+	var cases = []struct {
+		endpoint      string
+		expectederror string
+	}{
+		// case 0
+		{
+			endpoint:      "https://status-not-ok.io",
+			expectederror: "Unexpected server response: ",
+		},
+		// case 1
+		{
+			endpoint:      "https://invalid-json-response.io",
+			expectederror: "invalid json",
+		},
+		// case 2
+		{
+			endpoint:      "https://valid-batch-request-download.io",
+			expectederror: "",
+		},
+		// case 3
+		{
+			endpoint:      "https://response-no-objects.io",
+			expectederror: "",
+		},
+		// case 4
+		{
+			endpoint:      "https://unknown-transfer-adapter.io",
+			expectederror: "TransferAdapter not found: ",
+		},
+		// case 5
+		{
+			endpoint:      "https://error-in-response-objects.io",
+			expectederror: "Object not found",
+		},
+		// case 6
+		{
+			endpoint:      "https://empty-actions-map.io",
+			expectederror: "Missing action 'download'",
+		},
+		// case 7
+		{
+			endpoint:      "https://download-actions-map.io",
+			expectederror: "",
+		},
+		// case 8
+		{
+			endpoint:      "https://upload-actions-map.io",
+			expectederror: "Missing action 'download'",
+		},
+		// case 9
+		{
+			endpoint:      "https://verify-actions-map.io",
+			expectederror: "Missing action 'download'",
+		},
+		// case 10
+		{
+			endpoint:      "https://unknown-actions-map.io",
+			expectederror: "Missing action 'download'",
+		},
+	}
 
-			return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
+	for n, c := range cases {
+		client := &HTTPClient{
+			client:    hc,
+			endpoint:  c.endpoint,
+			transfers: make(map[string]TransferAdapter),
 		}
+		client.transfers["dummy"] = dummy
 
-		t.Errorf("Unknown test case: %s", url)
-
-		return nil
+		err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error {
+			if objectError != nil {
+				return objectError
+			}
+			b, err := io.ReadAll(content)
+			assert.NoError(t, err)
+			assert.Equal(t, []byte("dummy"), b)
+			return nil
+		})
+		if len(c.expectederror) > 0 {
+			assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
+		} else {
+			assert.NoError(t, err, "case %d", n)
+		}
 	}
+}
+
+func TestHTTPClientUpload(t *testing.T) {
+	p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6}
+
+	hc := &http.Client{Transport: RoundTripFunc(func(req *http.Request) *http.Response {
+		assert.Equal(t, "POST", req.Method)
+		assert.Equal(t, MediaType, req.Header.Get("Content-type"))
+		assert.Equal(t, MediaType, req.Header.Get("Accept"))
+
+		var batchRequest BatchRequest
+		err := jsoniter.NewDecoder(req.Body).Decode(&batchRequest)
+		assert.NoError(t, err)
 
-	hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)}
+		assert.Equal(t, "upload", batchRequest.Operation)
+		assert.Equal(t, 1, len(batchRequest.Objects))
+		assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid)
+		assert.Equal(t, p.Size, batchRequest.Objects[0].Size)
+
+		return lfsTestRoundtripHandler(req)
+	})}
 	dummy := &DummyTransferAdapter{}
 
 	var cases = []struct {
@@ -102,27 +286,57 @@ func TestHTTPClientDownload(t *testing.T) {
 		// case 0
 		{
 			endpoint:      "https://status-not-ok.io",
-			expectederror: "Unexpected servers response: ",
+			expectederror: "Unexpected server response: ",
 		},
 		// case 1
 		{
 			endpoint:      "https://invalid-json-response.io",
-			expectederror: "json.Decode: ",
+			expectederror: "invalid json",
 		},
 		// case 2
 		{
-			endpoint:      "https://valid-batch-request-download.io",
+			endpoint:      "https://valid-batch-request-upload.io",
 			expectederror: "",
 		},
 		// case 3
 		{
-			endpoint:      "https://invalid-response-no-objects.io",
-			expectederror: "No objects in result",
+			endpoint:      "https://response-no-objects.io",
+			expectederror: "",
 		},
 		// case 4
 		{
 			endpoint:      "https://unknown-transfer-adapter.io",
-			expectederror: "Transferadapter not found: ",
+			expectederror: "TransferAdapter not found: ",
+		},
+		// case 5
+		{
+			endpoint:      "https://error-in-response-objects.io",
+			expectederror: "Object not found",
+		},
+		// case 6
+		{
+			endpoint:      "https://empty-actions-map.io",
+			expectederror: "",
+		},
+		// case 7
+		{
+			endpoint:      "https://download-actions-map.io",
+			expectederror: "Missing action 'upload'",
+		},
+		// case 8
+		{
+			endpoint:      "https://upload-actions-map.io",
+			expectederror: "",
+		},
+		// case 9
+		{
+			endpoint:      "https://verify-actions-map.io",
+			expectederror: "Missing action 'upload'",
+		},
+		// case 10
+		{
+			endpoint:      "https://unknown-actions-map.io",
+			expectederror: "Missing action 'upload'",
 		},
 	}
 
@@ -134,7 +348,9 @@ func TestHTTPClientDownload(t *testing.T) {
 		}
 		client.transfers["dummy"] = dummy
 
-		_, err := client.Download(context.Background(), oid, size)
+		err := client.Upload(context.Background(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) {
+			return ioutil.NopCloser(new(bytes.Buffer)), objectError
+		})
 		if len(c.expectederror) > 0 {
 			assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
 		} else {