Add push to remote mirror repository (#15157)

* Added push mirror model.

* Integrated push mirror into queue.

* Moved methods into own file.

* Added basic implementation.

* Mirror wiki too.

* Removed duplicated method.

* Get url for different remotes.

* Added migration.

* Unified remote url access.

* Add/Remove push mirror remotes.

* Prevent hangs with missing credentials.

* Moved code between files.

* Changed sanitizer interface.

* Added push mirror backend methods.

* Only update the mirror remote.

* Limit refs on push.

* Added UI part.

* Added missing table.

* Delete mirror if repository gets removed.

* Changed signature. Handle object errors.

* Added upload method.

* Added "upload" unit tests.

* Added transfer adapter unit tests.

* Send correct headers.

* Added pushing of LFS objects.

* Added more logging.

* Simpler body handling.

* Process files in batches to reduce HTTP calls.

* Added created timestamp.

* Fixed invalid column name.

* Changed name to prevent xorm auto setting.

* Remove table header im empty.

* Strip exit code from error message.

* Added docs page about mirroring.

* Fixed date.

* Fixed merge errors.

* Moved test to integrations.

* Added push mirror test.

* Added test.

1 files changed, 132 insertions, 37 deletions

diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go
index 0eabd3faee..7dfdad417e 100644
--- a/modules/lfs/transferadapter_test.go
+++ b/modules/lfs/transferadapter_test.go
@@ -7,11 +7,13 @@ package lfs
 import (
 	"bytes"
 	"context"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"strings"
 	"testing"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -21,58 +23,151 @@ func TestBasicTransferAdapterName(t *testing.T) {
 	assert.Equal(t, "basic", a.Name())
 }
 
-func TestBasicTransferAdapterDownload(t *testing.T) {
+func TestBasicTransferAdapter(t *testing.T) {
+	p := Pointer{Oid: "b5a2c96250612366ea272ffac6d9744aaf4b45aacd96aa7cfcb931ee3b558259", Size: 5}
+
 	roundTripHandler := func(req *http.Request) *http.Response {
+		assert.Equal(t, MediaType, req.Header.Get("Accept"))
+		assert.Equal(t, "test-value", req.Header.Get("test-header"))
+
 		url := req.URL.String()
-		if strings.Contains(url, "valid-download-request") {
+		if strings.Contains(url, "download-request") {
 			assert.Equal(t, "GET", req.Method)
-			assert.Equal(t, "test-value", req.Header.Get("test-header"))
 
 			return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("dummy"))}
-		}
+		} else if strings.Contains(url, "upload-request") {
+			assert.Equal(t, "PUT", req.Method)
+			assert.Equal(t, "application/octet-stream", req.Header.Get("Content-Type"))
+
+			b, err := io.ReadAll(req.Body)
+			assert.NoError(t, err)
+			assert.Equal(t, "dummy", string(b))
 
-		t.Errorf("Unknown test case: %s", url)
+			return &http.Response{StatusCode: http.StatusOK}
+		} else if strings.Contains(url, "verify-request") {
+			assert.Equal(t, "POST", req.Method)
+			assert.Equal(t, MediaType, req.Header.Get("Content-Type"))
 
-		return nil
+			var vp Pointer
+			err := jsoniter.NewDecoder(req.Body).Decode(&vp)
+			assert.NoError(t, err)
+			assert.Equal(t, p.Oid, vp.Oid)
+			assert.Equal(t, p.Size, vp.Size)
+
+			return &http.Response{StatusCode: http.StatusOK}
+		} else if strings.Contains(url, "error-response") {
+			er := &ErrorResponse{
+				Message: "Object not found",
+			}
+			payload := new(bytes.Buffer)
+			jsoniter.NewEncoder(payload).Encode(er)
+
+			return &http.Response{StatusCode: http.StatusNotFound, Body: ioutil.NopCloser(payload)}
+		} else {
+			t.Errorf("Unknown test case: %s", url)
+			return nil
+		}
 	}
 
 	hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)}
 	a := &BasicTransferAdapter{hc}
 
-	var cases = []struct {
-		response      *ObjectResponse
-		expectederror string
-	}{
-		// case 0
-		{
-			response:      &ObjectResponse{},
-			expectederror: "Action 'download' not found",
-		},
-		// case 1
-		{
-			response: &ObjectResponse{
-				Actions: map[string]*Link{"upload": nil},
+	t.Run("Download", func(t *testing.T) {
+		cases := []struct {
+			link          *Link
+			expectederror string
+		}{
+			// case 0
+			{
+				link: &Link{
+					Href:   "https://download-request.io",
+					Header: map[string]string{"test-header": "test-value"},
+				},
+				expectederror: "",
 			},
-			expectederror: "Action 'download' not found",
-		},
-		// case 2
-		{
-			response: &ObjectResponse{
-				Actions: map[string]*Link{"download": {
-					Href:   "https://valid-download-request.io",
+			// case 1
+			{
+				link: &Link{
+					Href:   "https://error-response.io",
 					Header: map[string]string{"test-header": "test-value"},
-				}},
+				},
+				expectederror: "Object not found",
 			},
-			expectederror: "",
-		},
-	}
+		}
 
-	for n, c := range cases {
-		_, err := a.Download(context.Background(), c.response)
-		if len(c.expectederror) > 0 {
-			assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
-		} else {
-			assert.NoError(t, err, "case %d", n)
+		for n, c := range cases {
+			_, err := a.Download(context.Background(), c.link)
+			if len(c.expectederror) > 0 {
+				assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
+			} else {
+				assert.NoError(t, err, "case %d", n)
+			}
 		}
-	}
+	})
+
+	t.Run("Upload", func(t *testing.T) {
+		cases := []struct {
+			link          *Link
+			expectederror string
+		}{
+			// case 0
+			{
+				link: &Link{
+					Href:   "https://upload-request.io",
+					Header: map[string]string{"test-header": "test-value"},
+				},
+				expectederror: "",
+			},
+			// case 1
+			{
+				link: &Link{
+					Href:   "https://error-response.io",
+					Header: map[string]string{"test-header": "test-value"},
+				},
+				expectederror: "Object not found",
+			},
+		}
+
+		for n, c := range cases {
+			err := a.Upload(context.Background(), c.link, p, bytes.NewBufferString("dummy"))
+			if len(c.expectederror) > 0 {
+				assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
+			} else {
+				assert.NoError(t, err, "case %d", n)
+			}
+		}
+	})
+
+	t.Run("Verify", func(t *testing.T) {
+		cases := []struct {
+			link          *Link
+			expectederror string
+		}{
+			// case 0
+			{
+				link: &Link{
+					Href:   "https://verify-request.io",
+					Header: map[string]string{"test-header": "test-value"},
+				},
+				expectederror: "",
+			},
+			// case 1
+			{
+				link: &Link{
+					Href:   "https://error-response.io",
+					Header: map[string]string{"test-header": "test-value"},
+				},
+				expectederror: "Object not found",
+			},
+		}
+
+		for n, c := range cases {
+			err := a.Verify(context.Background(), c.link, p)
+			if len(c.expectederror) > 0 {
+				assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
+			} else {
+				assert.NoError(t, err, "case %d", n)
+			}
+		}
+	})
 }