Move migrating repository from frontend to backend (#6200)

* move migrating to backend

* add loading image when migrating and fix tests

* fix format

* fix lint

* add redis task queue support and improve docs

* add redis vendor

* fix vet

* add database migrations and fix app.ini sample

* add comments for task section on app.ini.sample

* Update models/migrations/v84.go

Co-Authored-By: lunny <xiaolunwen@gmail.com>

* Update models/repo.go

Co-Authored-By: lunny <xiaolunwen@gmail.com>

* move migrating to backend

* add loading image when migrating and fix tests

* fix fmt

* add redis task queue support and improve docs

* fix fixtures

* fix fixtures

* fix duplicate function on index.js

* fix tests

* rename repository statuses

* check if repository is being create when SSH request

* fix lint

* fix template

* some improvements

* fix template

* unified migrate options

* fix lint

* fix loading page

* refactor

* When gitea restart, don't restart the running tasks because we may have servel gitea instances, that may break the migration

* fix js

* Update models/repo.go

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Update docs/content/doc/advanced/config-cheat-sheet.en-us.md

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>

* fix tests

* rename ErrTaskIsNotExist to ErrTaskDoesNotExist

* delete release after add one on tests to make it run happy

* fix tests

* fix tests

* improve codes

* fix lint

* fix lint

* fix migrations

6 files changed, 109 insertions, 43 deletions

diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go
index d8b06862a5..08c0635bc3 100644
--- a/routers/api/v1/repo/repo.go
+++ b/routers/api/v1/repo/repo.go
@@ -398,8 +398,8 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
 	}
 
 	var opts = migrations.MigrateOptions{
-		RemoteURL:    remoteAddr,
-		Name:         form.RepoName,
+		CloneAddr:    remoteAddr,
+		RepoName:     form.RepoName,
 		Description:  form.Description,
 		Private:      form.Private || setting.Repository.ForcePrivate,
 		Mirror:       form.Mirror,
diff --git a/routers/init.go b/routers/init.go
index 1efddcfaa6..c37bbeb6b0 100644
--- a/routers/init.go
+++ b/routers/init.go
@@ -20,6 +20,7 @@ import (
 	"code.gitea.io/gitea/modules/markup/external"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/ssh"
+	"code.gitea.io/gitea/modules/task"
 	"code.gitea.io/gitea/services/mailer"
 	mirror_service "code.gitea.io/gitea/services/mirror"
 
@@ -102,6 +103,9 @@ func GlobalInit() {
 		mirror_service.InitSyncMirrors()
 		models.InitDeliverHooks()
 		models.InitTestPullRequests()
+		if err := task.Init(); err != nil {
+			log.Fatal("Failed to initialize task scheduler: %v", err)
+		}
 	}
 	if setting.EnableSQLite3 {
 		log.Info("SQLite3 Supported")
diff --git a/routers/private/serv.go b/routers/private/serv.go
index 71c0f6ea2c..c4508b4cb5 100644
--- a/routers/private/serv.go
+++ b/routers/private/serv.go
@@ -119,6 +119,15 @@ func ServCommand(ctx *macaron.Context) {
 	repo.OwnerName = ownerName
 	results.RepoID = repo.ID
 
+	if repo.IsBeingCreated() {
+		ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
+			"results": results,
+			"type":    "InternalServerError",
+			"err":     "Repository is being created, you could retry after it finished",
+		})
+		return
+	}
+
 	// We can shortcut at this point if the repo is a mirror
 	if mode > models.AccessModeRead && repo.IsMirror {
 		ctx.JSON(http.StatusUnauthorized, map[string]interface{}{
diff --git a/routers/repo/repo.go b/routers/repo/repo.go
index b67384d721..bfd0c771b0 100644
--- a/routers/repo/repo.go
+++ b/routers/repo/repo.go
@@ -19,6 +19,7 @@ import (
 	"code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/notification"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/task"
 	"code.gitea.io/gitea/modules/util"
 
 	"github.com/unknwon/com"
@@ -133,8 +134,6 @@ func Create(ctx *context.Context) {
 
 func handleCreateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form interface{}) {
 	switch {
-	case migrations.IsRateLimitError(err):
-		ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form)
 	case models.IsErrReachLimitOfRepo(err):
 		ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form)
 	case models.IsErrRepoAlreadyExist(err):
@@ -221,6 +220,40 @@ func Migrate(ctx *context.Context) {
 	ctx.HTML(200, tplMigrate)
 }
 
+func handleMigrateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form *auth.MigrateRepoForm) {
+	switch {
+	case migrations.IsRateLimitError(err):
+		ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form)
+	case migrations.IsTwoFactorAuthError(err):
+		ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tpl, form)
+	case models.IsErrReachLimitOfRepo(err):
+		ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form)
+	case models.IsErrRepoAlreadyExist(err):
+		ctx.Data["Err_RepoName"] = true
+		ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tpl, form)
+	case models.IsErrNameReserved(err):
+		ctx.Data["Err_RepoName"] = true
+		ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tpl, form)
+	case models.IsErrNamePatternNotAllowed(err):
+		ctx.Data["Err_RepoName"] = true
+		ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form)
+	default:
+		remoteAddr, _ := form.ParseRemoteAddr(owner)
+		err = util.URLSanitizedError(err, remoteAddr)
+		if strings.Contains(err.Error(), "Authentication failed") ||
+			strings.Contains(err.Error(), "Bad credentials") ||
+			strings.Contains(err.Error(), "could not read Username") {
+			ctx.Data["Err_Auth"] = true
+			ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tpl, form)
+		} else if strings.Contains(err.Error(), "fatal:") {
+			ctx.Data["Err_CloneAddr"] = true
+			ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tpl, form)
+		} else {
+			ctx.ServerError(name, err)
+		}
+	}
+}
+
 // MigratePost response for migrating from external git repository
 func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) {
 	ctx.Data["Title"] = ctx.Tr("new_migrate")
@@ -258,8 +291,8 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) {
 	}
 
 	var opts = migrations.MigrateOptions{
-		RemoteURL:    remoteAddr,
-		Name:         form.RepoName,
+		CloneAddr:    remoteAddr,
+		RepoName:     form.RepoName,
 		Description:  form.Description,
 		Private:      form.Private || setting.Repository.ForcePrivate,
 		Mirror:       form.Mirror,
@@ -282,47 +315,19 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) {
 		opts.Releases = false
 	}
 
-	repo, err := migrations.MigrateRepository(ctx.User, ctxUser.Name, opts)
-	if err == nil {
-		notification.NotifyCreateRepository(ctx.User, ctxUser, repo)
-
-		log.Trace("Repository migrated [%d]: %s/%s successfully", repo.ID, ctxUser.Name, form.RepoName)
-		ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + form.RepoName)
+	err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName)
+	if err != nil {
+		handleMigrateError(ctx, ctxUser, err, "MigratePost", tplMigrate, &form)
 		return
 	}
 
-	switch {
-	case models.IsErrReachLimitOfRepo(err):
-		ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", ctxUser.MaxCreationLimit()), tplMigrate, &form)
-	case models.IsErrNameReserved(err):
-		ctx.Data["Err_RepoName"] = true
-		ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tplMigrate, &form)
-	case models.IsErrRepoAlreadyExist(err):
-		ctx.Data["Err_RepoName"] = true
-		ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tplMigrate, &form)
-	case models.IsErrNamePatternNotAllowed(err):
-		ctx.Data["Err_RepoName"] = true
-		ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tplMigrate, &form)
-	case migrations.IsRateLimitError(err):
-		ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tplMigrate, &form)
-	case migrations.IsTwoFactorAuthError(err):
-		ctx.Data["Err_Auth"] = true
-		ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tplMigrate, &form)
-	default:
-		// remoteAddr may contain credentials, so we sanitize it
-		err = util.URLSanitizedError(err, remoteAddr)
-		if strings.Contains(err.Error(), "Authentication failed") ||
-			strings.Contains(err.Error(), "Bad credentials") ||
-			strings.Contains(err.Error(), "could not read Username") {
-			ctx.Data["Err_Auth"] = true
-			ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tplMigrate, &form)
-		} else if strings.Contains(err.Error(), "fatal:") {
-			ctx.Data["Err_CloneAddr"] = true
-			ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tplMigrate, &form)
-		} else {
-			ctx.ServerError("MigratePost", err)
-		}
+	err = task.MigrateRepository(ctx.User, ctxUser, opts)
+	if err == nil {
+		ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + opts.RepoName)
+		return
 	}
+
+	handleMigrateError(ctx, ctxUser, err, "MigratePost", tplMigrate, &form)
 }
 
 // Action response for actions to a repository
@@ -460,3 +465,19 @@ func Download(ctx *context.Context) {
 
 	ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext)
 }
+
+// Status returns repository's status
+func Status(ctx *context.Context) {
+	task, err := models.GetMigratingTask(ctx.Repo.Repository.ID)
+	if err != nil {
+		ctx.JSON(500, map[string]interface{}{
+			"err": err,
+		})
+		return
+	}
+
+	ctx.JSON(200, map[string]interface{}{
+		"status": ctx.Repo.Repository.Status,
+		"err":    task.Errors,
+	})
+}
diff --git a/routers/repo/view.go b/routers/repo/view.go
index 1967b511ca..c4e6a69220 100644
--- a/routers/repo/view.go
+++ b/routers/repo/view.go
@@ -11,6 +11,7 @@ import (
 	"fmt"
 	gotemplate "html/template"
 	"io/ioutil"
+	"net/url"
 	"path"
 	"strings"
 
@@ -31,6 +32,7 @@ const (
 	tplRepoHome  base.TplName = "repo/home"
 	tplWatchers  base.TplName = "repo/watchers"
 	tplForks     base.TplName = "repo/forks"
+	tplMigrating base.TplName = "repo/migrating"
 )
 
 func renderDirectory(ctx *context.Context, treeLink string) {
@@ -356,9 +358,37 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
 	}
 }
 
+func safeURL(address string) string {
+	u, err := url.Parse(address)
+	if err != nil {
+		return address
+	}
+	u.User = nil
+	return u.String()
+}
+
 // Home render repository home page
 func Home(ctx *context.Context) {
 	if len(ctx.Repo.Units) > 0 {
+		if ctx.Repo.Repository.IsBeingCreated() {
+			task, err := models.GetMigratingTask(ctx.Repo.Repository.ID)
+			if err != nil {
+				ctx.ServerError("models.GetMigratingTask", err)
+				return
+			}
+			cfg, err := task.MigrateConfig()
+			if err != nil {
+				ctx.ServerError("task.MigrateConfig", err)
+				return
+			}
+
+			ctx.Data["Repo"] = ctx.Repo
+			ctx.Data["MigrateTask"] = task
+			ctx.Data["CloneAddr"] = safeURL(cfg.CloneAddr)
+			ctx.HTML(200, tplMigrating)
+			return
+		}
+
 		var firstUnit *models.Unit
 		for _, repoUnit := range ctx.Repo.Units {
 			if repoUnit.Type == models.UnitTypeCode {
diff --git a/routers/routes/routes.go b/routers/routes/routes.go
index 11f2029226..8dfcdb9c9b 100644
--- a/routers/routes/routes.go
+++ b/routers/routes/routes.go
@@ -845,6 +845,8 @@ func RegisterRoutes(m *macaron.Macaron) {
 
 		m.Get("/archive/*", repo.MustBeNotEmpty, reqRepoCodeReader, repo.Download)
 
+		m.Get("/status", reqRepoCodeReader, repo.Status)
+
 		m.Group("/branches", func() {
 			m.Get("", repo.Branches)
 		}, repo.MustBeNotEmpty, context.RepoRef(), reqRepoCodeReader)