Add push to remote mirror repository (#15157)

* Added push mirror model.

* Integrated push mirror into queue.

* Moved methods into own file.

* Added basic implementation.

* Mirror wiki too.

* Removed duplicated method.

* Get url for different remotes.

* Added migration.

* Unified remote url access.

* Add/Remove push mirror remotes.

* Prevent hangs with missing credentials.

* Moved code between files.

* Changed sanitizer interface.

* Added push mirror backend methods.

* Only update the mirror remote.

* Limit refs on push.

* Added UI part.

* Added missing table.

* Delete mirror if repository gets removed.

* Changed signature. Handle object errors.

* Added upload method.

* Added "upload" unit tests.

* Added transfer adapter unit tests.

* Send correct headers.

* Added pushing of LFS objects.

* Added more logging.

* Simpler body handling.

* Process files in batches to reduce HTTP calls.

* Added created timestamp.

* Fixed invalid column name.

* Changed name to prevent xorm auto setting.

* Remove table header im empty.

* Strip exit code from error message.

* Added docs page about mirroring.

* Fixed date.

* Fixed merge errors.

* Moved test to integrations.

* Added push mirror test.

* Added test.

2 files changed, 178 insertions, 0 deletions

diff --git a/integrations/mirror_pull_test.go b/integrations/mirror_pull_test.go
new file mode 100644
index 0000000000..0e4da74fcf
--- /dev/null
+++ b/integrations/mirror_pull_test.go
@@ -0,0 +1,92 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"context"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/git"
+	migration "code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/repository"
+	mirror_service "code.gitea.io/gitea/services/mirror"
+	release_service "code.gitea.io/gitea/services/release"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMirrorPull(t *testing.T) {
+	defer prepareTestEnv(t)()
+
+	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
+	repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
+	repoPath := models.RepoPath(user.Name, repo.Name)
+
+	opts := migration.MigrateOptions{
+		RepoName:    "test_mirror",
+		Description: "Test mirror",
+		Private:     false,
+		Mirror:      true,
+		CloneAddr:   repoPath,
+		Wiki:        true,
+		Releases:    false,
+	}
+
+	mirrorRepo, err := repository.CreateRepository(user, user, models.CreateRepoOptions{
+		Name:        opts.RepoName,
+		Description: opts.Description,
+		IsPrivate:   opts.Private,
+		IsMirror:    opts.Mirror,
+		Status:      models.RepositoryBeingMigrated,
+	})
+	assert.NoError(t, err)
+
+	ctx := context.Background()
+
+	mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts)
+	assert.NoError(t, err)
+
+	gitRepo, err := git.OpenRepository(repoPath)
+	assert.NoError(t, err)
+	defer gitRepo.Close()
+
+	findOptions := models.FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
+	initCount, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions)
+	assert.NoError(t, err)
+
+	assert.NoError(t, release_service.CreateRelease(gitRepo, &models.Release{
+		RepoID:       repo.ID,
+		PublisherID:  user.ID,
+		TagName:      "v0.2",
+		Target:       "master",
+		Title:        "v0.2 is released",
+		Note:         "v0.2 is released",
+		IsDraft:      false,
+		IsPrerelease: false,
+		IsTag:        true,
+	}, nil, ""))
+
+	err = mirror.GetMirror()
+	assert.NoError(t, err)
+
+	ok := mirror_service.SyncPullMirror(ctx, mirror.ID)
+	assert.True(t, ok)
+
+	count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions)
+	assert.NoError(t, err)
+	assert.EqualValues(t, initCount+1, count)
+
+	release, err := models.GetRelease(repo.ID, "v0.2")
+	assert.NoError(t, err)
+	assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true))
+
+	ok = mirror_service.SyncPullMirror(ctx, mirror.ID)
+	assert.True(t, ok)
+
+	count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions)
+	assert.NoError(t, err)
+	assert.EqualValues(t, initCount, count)
+}
diff --git a/integrations/mirror_push_test.go b/integrations/mirror_push_test.go
new file mode 100644
index 0000000000..3191ef7704
--- /dev/null
+++ b/integrations/mirror_push_test.go
@@ -0,0 +1,86 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/repository"
+	"code.gitea.io/gitea/modules/setting"
+	mirror_service "code.gitea.io/gitea/services/mirror"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMirrorPush(t *testing.T) {
+	onGiteaRun(t, testMirrorPush)
+}
+
+func testMirrorPush(t *testing.T, u *url.URL) {
+	defer prepareTestEnv(t)()
+
+	setting.Migrations.AllowLocalNetworks = true
+
+	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
+	srcRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
+
+	mirrorRepo, err := repository.CreateRepository(user, user, models.CreateRepoOptions{
+		Name: "test-push-mirror",
+	})
+	assert.NoError(t, err)
+
+	ctx := NewAPITestContext(t, user.LowerName, srcRepo.Name)
+
+	doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword)(t)
+
+	mirrors, err := models.GetPushMirrorsByRepoID(srcRepo.ID)
+	assert.NoError(t, err)
+	assert.Len(t, mirrors, 1)
+
+	ok := mirror_service.SyncPushMirror(context.Background(), mirrors[0].ID)
+	assert.True(t, ok)
+
+	srcGitRepo, err := git.OpenRepository(srcRepo.RepoPath())
+	assert.NoError(t, err)
+	defer srcGitRepo.Close()
+
+	srcCommit, err := srcGitRepo.GetBranchCommit("master")
+	assert.NoError(t, err)
+
+	mirrorGitRepo, err := git.OpenRepository(mirrorRepo.RepoPath())
+	assert.NoError(t, err)
+	defer mirrorGitRepo.Close()
+
+	mirrorCommit, err := mirrorGitRepo.GetBranchCommit("master")
+	assert.NoError(t, err)
+
+	assert.Equal(t, srcCommit.ID, mirrorCommit.ID)
+}
+
+func doCreatePushMirror(ctx APITestContext, address, username, password string) func(t *testing.T) {
+	return func(t *testing.T) {
+		csrf := GetCSRF(t, ctx.Session, fmt.Sprintf("/%s/%s/settings", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame)))
+
+		req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/settings", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame)), map[string]string{
+			"_csrf":                csrf,
+			"action":               "push-mirror-add",
+			"push_mirror_address":  address,
+			"push_mirror_username": username,
+			"push_mirror_password": password,
+			"push_mirror_interval": "0",
+		})
+		ctx.Session.MakeRequest(t, req, http.StatusFound)
+
+		flashCookie := ctx.Session.GetCookie("macaron_flash")
+		assert.NotNil(t, flashCookie)
+		assert.Contains(t, flashCookie.Value, "success")
+	}
+}