Add push to remote mirror repository (#15157)

* Added push mirror model.

* Integrated push mirror into queue.

* Moved methods into own file.

* Added basic implementation.

* Mirror wiki too.

* Removed duplicated method.

* Get url for different remotes.

* Added migration.

* Unified remote url access.

* Add/Remove push mirror remotes.

* Prevent hangs with missing credentials.

* Moved code between files.

* Changed sanitizer interface.

* Added push mirror backend methods.

* Only update the mirror remote.

* Limit refs on push.

* Added UI part.

* Added missing table.

* Delete mirror if repository gets removed.

* Changed signature. Handle object errors.

* Added upload method.

* Added "upload" unit tests.

* Added transfer adapter unit tests.

* Send correct headers.

* Added pushing of LFS objects.

* Added more logging.

* Simpler body handling.

* Process files in batches to reduce HTTP calls.

* Added created timestamp.

* Fixed invalid column name.

* Changed name to prevent xorm auto setting.

* Remove table header im empty.

* Strip exit code from error message.

* Added docs page about mirroring.

* Fixed date.

* Fixed merge errors.

* Moved test to integrations.

* Added push mirror test.

* Added test.

1 files changed, 53 insertions, 10 deletions

diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go
index 3a51564a82..dc72981a9e 100644
--- a/modules/lfs/filesystem_client.go
+++ b/modules/lfs/filesystem_client.go
@@ -19,6 +19,11 @@ type FilesystemClient struct {
 	lfsdir string
 }
 
+// BatchSize returns the preferred size of batchs to process
+func (c *FilesystemClient) BatchSize() int {
+	return 1
+}
+
 func newFilesystemClient(endpoint *url.URL) *FilesystemClient {
 	path, _ := util.FileURLToPath(endpoint)
 
@@ -33,18 +38,56 @@ func (c *FilesystemClient) objectPath(oid string) string {
 	return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid)
 }
 
-// Download reads the specific LFS object from the target repository
-func (c *FilesystemClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) {
-	objectPath := c.objectPath(oid)
+// Download reads the specific LFS object from the target path
+func (c *FilesystemClient) Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error {
+	for _, object := range objects {
+		p := Pointer{object.Oid, object.Size}
 
-	if _, err := os.Stat(objectPath); os.IsNotExist(err) {
-		return nil, err
-	}
+		objectPath := c.objectPath(p.Oid)
+
+		f, err := os.Open(objectPath)
+		if err != nil {
+			return err
+		}
 
-	file, err := os.Open(objectPath)
-	if err != nil {
-		return nil, err
+		if err := callback(p, f, nil); err != nil {
+			return err
+		}
 	}
+	return nil
+}
+
+// Upload writes the specific LFS object to the target path
+func (c *FilesystemClient) Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error {
+	for _, object := range objects {
+		p := Pointer{object.Oid, object.Size}
+
+		objectPath := c.objectPath(p.Oid)
 
-	return file, nil
+		if err := os.MkdirAll(filepath.Dir(objectPath), os.ModePerm); err != nil {
+			return err
+		}
+
+		content, err := callback(p, nil)
+		if err != nil {
+			return err
+		}
+
+		err = func() error {
+			defer content.Close()
+
+			f, err := os.Create(objectPath)
+			if err != nil {
+				return err
+			}
+
+			_, err = io.Copy(f, content)
+
+			return err
+		}()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }