* Add migrating message Signed-off-by: Andrew Thornton <art27@cantab.net> * simplify messenger Signed-off-by: Andrew Thornton <art27@cantab.net> * make messenger an interface Signed-off-by: Andrew Thornton <art27@cantab.net> * rename Signed-off-by: Andrew Thornton <art27@cantab.net> * prepare for merge Signed-off-by: Andrew Thornton <art27@cantab.net> * as per tech Signed-off-by: Andrew Thornton <art27@cantab.net> Co-authored-by: 6543 <6543@obermui.de>tags/v1.15.0-rc1
@@ -317,6 +317,8 @@ var migrations = []Migration{ | |||
NewMigration("Add issue resource index table", addIssueResourceIndexTable), | |||
// v183 -> v184 | |||
NewMigration("Create PushMirror table", createPushMirrorTable), | |||
// v184 -> v185 | |||
NewMigration("Rename Task errors to message", renameTaskErrorsToMessage), | |||
} | |||
// GetCurrentDBVersion returns the current db version |
@@ -0,0 +1,47 @@ | |||
// Copyright 2021 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
package migrations | |||
import ( | |||
"fmt" | |||
"code.gitea.io/gitea/modules/setting" | |||
"xorm.io/xorm" | |||
) | |||
func renameTaskErrorsToMessage(x *xorm.Engine) error { | |||
type Task struct { | |||
Errors string `xorm:"TEXT"` // if task failed, saved the error reason | |||
Type int | |||
Status int `xorm:"index"` | |||
} | |||
sess := x.NewSession() | |||
defer sess.Close() | |||
if err := sess.Begin(); err != nil { | |||
return err | |||
} | |||
if err := sess.Sync2(new(Task)); err != nil { | |||
return fmt.Errorf("error on Sync2: %v", err) | |||
} | |||
switch { | |||
case setting.Database.UseMySQL: | |||
if _, err := sess.Exec("ALTER TABLE `task` CHANGE errors message text"); err != nil { | |||
return err | |||
} | |||
case setting.Database.UseMSSQL: | |||
if _, err := sess.Exec("sp_rename 'task.errors', 'message', 'COLUMN'"); err != nil { | |||
return err | |||
} | |||
default: | |||
if _, err := sess.Exec("ALTER TABLE `task` RENAME COLUMN errors TO message"); err != nil { | |||
return err | |||
} | |||
} | |||
return sess.Commit() | |||
} |
@@ -32,10 +32,16 @@ type Task struct { | |||
StartTime timeutil.TimeStamp | |||
EndTime timeutil.TimeStamp | |||
PayloadContent string `xorm:"TEXT"` | |||
Errors string `xorm:"TEXT"` // if task failed, saved the error reason | |||
Message string `xorm:"TEXT"` // if task failed, saved the error reason | |||
Created timeutil.TimeStamp `xorm:"created"` | |||
} | |||
// TranslatableMessage represents JSON struct that can be translated with a Locale | |||
type TranslatableMessage struct { | |||
Format string | |||
Args []interface{} `json:"omitempty"` | |||
} | |||
// LoadRepo loads repository of the task | |||
func (task *Task) LoadRepo() error { | |||
return task.loadRepo(x) |
@@ -0,0 +1,11 @@ | |||
// Copyright 2021 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
package base | |||
// Messenger is a formatting function similar to i18n.Tr | |||
type Messenger func(key string, args ...interface{}) | |||
// NilMessenger represents an empty formatting function | |||
func NilMessenger(string, ...interface{}) {} |
@@ -555,7 +555,7 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi | |||
return err | |||
} | |||
if err := migrateRepository(downloader, uploader, opts); err != nil { | |||
if err := migrateRepository(downloader, uploader, opts, nil); err != nil { | |||
if err1 := uploader.Rollback(); err1 != nil { | |||
log.Error("rollback failed: %v", err1) | |||
} | |||
@@ -620,7 +620,7 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName | |||
} | |||
updateOptionsUnits(&migrateOpts, units) | |||
if err = migrateRepository(downloader, uploader, migrateOpts); err != nil { | |||
if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil { | |||
if err1 := uploader.Rollback(); err1 != nil { | |||
log.Error("rollback failed: %v", err1) | |||
} |
@@ -47,7 +47,7 @@ func TestGiteaUploadRepo(t *testing.T) { | |||
PullRequests: true, | |||
Private: true, | |||
Mirror: false, | |||
}) | |||
}, nil) | |||
assert.NoError(t, err) | |||
repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository) |
@@ -99,7 +99,7 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error { | |||
} | |||
// MigrateRepository migrate repository according MigrateOptions | |||
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { | |||
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*models.Repository, error) { | |||
err := IsMigrateURLAllowed(opts.CloneAddr, doer) | |||
if err != nil { | |||
return nil, err | |||
@@ -118,7 +118,7 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, | |||
var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) | |||
uploader.gitServiceType = opts.GitServiceType | |||
if err := migrateRepository(downloader, uploader, opts); err != nil { | |||
if err := migrateRepository(downloader, uploader, opts, messenger); err != nil { | |||
if err1 := uploader.Rollback(); err1 != nil { | |||
log.Error("rollback failed: %v", err1) | |||
} | |||
@@ -167,7 +167,11 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio | |||
// migrateRepository will download information and then upload it to Uploader, this is a simple | |||
// process for small repository. For a big repository, save all the data to disk | |||
// before upload is better | |||
func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error { | |||
func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error { | |||
if messenger == nil { | |||
messenger = base.NilMessenger | |||
} | |||
repo, err := downloader.GetRepoInfo() | |||
if err != nil { | |||
if !base.IsErrNotSupported(err) { | |||
@@ -185,12 +189,14 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts | |||
} | |||
log.Trace("migrating git data from %s", repo.CloneURL) | |||
messenger("repo.migrate.migrating_git") | |||
if err = uploader.CreateRepo(repo, opts); err != nil { | |||
return err | |||
} | |||
defer uploader.Close() | |||
log.Trace("migrating topics") | |||
messenger("repo.migrate.migrating_topics") | |||
topics, err := downloader.GetTopics() | |||
if err != nil { | |||
if !base.IsErrNotSupported(err) { | |||
@@ -206,6 +212,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts | |||
if opts.Milestones { | |||
log.Trace("migrating milestones") | |||
messenger("repo.migrate.migrating_milestones") | |||
milestones, err := downloader.GetMilestones() | |||
if err != nil { | |||
if !base.IsErrNotSupported(err) { | |||
@@ -229,6 +236,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts | |||
if opts.Labels { | |||
log.Trace("migrating labels") | |||
messenger("repo.migrate.migrating_labels") | |||
labels, err := downloader.GetLabels() | |||
if err != nil { | |||
if !base.IsErrNotSupported(err) { | |||
@@ -252,6 +260,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts | |||
if opts.Releases { | |||
log.Trace("migrating releases") | |||
messenger("repo.migrate.migrating_releases") | |||
releases, err := downloader.GetReleases() | |||
if err != nil { | |||
if !base.IsErrNotSupported(err) { | |||
@@ -285,6 +294,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts | |||
if opts.Issues { | |||
log.Trace("migrating issues and comments") | |||
messenger("repo.migrate.migrating_issues") | |||
var issueBatchSize = uploader.MaxBatchInsertSize("issue") | |||
for i := 1; ; i++ { | |||
@@ -339,6 +349,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts | |||
if opts.PullRequests { | |||
log.Trace("migrating pull requests and comments") | |||
messenger("repo.migrate.migrating_pulls") | |||
var prBatchSize = uploader.MaxBatchInsertSize("pullrequest") | |||
for i := 1; ; i++ { | |||
prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize) |
@@ -20,6 +20,7 @@ import ( | |||
"code.gitea.io/gitea/modules/structs" | |||
"code.gitea.io/gitea/modules/timeutil" | |||
"code.gitea.io/gitea/modules/util" | |||
jsoniter "github.com/json-iterator/go" | |||
) | |||
func handleCreateError(owner *models.User, err error) error { | |||
@@ -56,7 +57,7 @@ func runMigrateTask(t *models.Task) (err error) { | |||
t.EndTime = timeutil.TimeStampNow() | |||
t.Status = structs.TaskStatusFailed | |||
t.Errors = err.Error() | |||
t.Message = err.Error() | |||
t.RepoID = 0 | |||
if err := t.UpdateCols("status", "errors", "repo_id", "end_time"); err != nil { | |||
log.Error("Task UpdateCols failed: %v", err) | |||
@@ -106,7 +107,16 @@ func runMigrateTask(t *models.Task) (err error) { | |||
return | |||
} | |||
repo, err = migrations.MigrateRepository(ctx, t.Doer, t.Owner.Name, *opts) | |||
repo, err = migrations.MigrateRepository(ctx, t.Doer, t.Owner.Name, *opts, func(format string, args ...interface{}) { | |||
message := models.TranslatableMessage{ | |||
Format: format, | |||
Args: args, | |||
} | |||
json := jsoniter.ConfigCompatibleWithStandardLibrary | |||
bs, _ := json.Marshal(message) | |||
t.Message = string(bs) | |||
_ = t.UpdateCols("message") | |||
}) | |||
if err == nil { | |||
log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) | |||
return |
@@ -824,11 +824,19 @@ migrated_from_fake = Migrated From %[1]s | |||
migrate.migrate = Migrate From %s | |||
migrate.migrating = Migrating from <b>%s</b> ... | |||
migrate.migrating_failed = Migrating from <b>%s</b> failed. | |||
migrate.migrating_failed.error = Error: %s | |||
migrate.github.description = Migrating data from Github.com or Github Enterprise. | |||
migrate.git.description = Migrating or Mirroring git data from Git services | |||
migrate.gitlab.description = Migrating data from GitLab.com or Self-Hosted gitlab server. | |||
migrate.gitea.description = Migrating data from Gitea.com or Self-Hosted Gitea server. | |||
migrate.gogs.description = Migrating data from notabug.org or other Self-Hosted Gogs server. | |||
migrate.migrating_git = Migrating Git Data | |||
migrate.migrating_topics = Migrating Topics | |||
migrate.migrating_milestones = Migrating Milestones | |||
migrate.migrating_labels = Migrating Labels | |||
migrate.migrating_releases = Migrating Releases | |||
migrate.migrating_issues = Migrating Issues | |||
migrate.migrating_pulls = Migrating Pull Requests | |||
mirror_from = mirror of | |||
forked_from = forked from |
@@ -199,7 +199,7 @@ func Migrate(ctx *context.APIContext) { | |||
} | |||
}() | |||
if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, repoOwner.Name, opts); err != nil { | |||
if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, repoOwner.Name, opts, nil); err != nil { | |||
handleMigrateError(ctx, repoOwner, remoteAddr, err) | |||
return | |||
} |
@@ -9,6 +9,7 @@ import ( | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/context" | |||
jsoniter "github.com/json-iterator/go" | |||
) | |||
// TaskStatus returns task's status | |||
@@ -21,9 +22,24 @@ func TaskStatus(ctx *context.Context) { | |||
return | |||
} | |||
message := task.Message | |||
if task.Message != "" && task.Message[0] == '{' { | |||
// assume message is actually a translatable string | |||
json := jsoniter.ConfigCompatibleWithStandardLibrary | |||
var translatableMessage models.TranslatableMessage | |||
if err := json.Unmarshal([]byte(message), &translatableMessage); err != nil { | |||
translatableMessage = models.TranslatableMessage{ | |||
Format: "migrate.migrating_failed.error", | |||
Args: []interface{}{task.Message}, | |||
} | |||
} | |||
message = ctx.Tr(translatableMessage.Format, translatableMessage.Args...) | |||
} | |||
ctx.JSON(http.StatusOK, map[string]interface{}{ | |||
"status": task.Status, | |||
"err": task.Errors, | |||
"message": message, | |||
"repo-id": task.RepoID, | |||
"repo-name": opts.RepoName, | |||
"start": task.StartTime, |
@@ -22,6 +22,7 @@ | |||
<div class="sixteen wide center aligned centered column"> | |||
<div id="repo_migrating_progress"> | |||
<p>{{.i18n.Tr "repo.migrate.migrating" .CloneAddr | Safe}}</p> | |||
<p id="repo_migrating_progress_message"></p> | |||
</div> | |||
<div id="repo_migrating_failed" hidden> | |||
<p>{{.i18n.Tr "repo.migrate.migrating_failed" .CloneAddr | Safe}}</p> |
@@ -202,6 +202,7 @@ function initRepoStatusChecker() { | |||
const migrating = $('#repo_migrating'); | |||
$('#repo_migrating_failed').hide(); | |||
$('#repo_migrating_failed_image').hide(); | |||
$('#repo_migrating_progress_message').hide(); | |||
if (migrating) { | |||
const task = migrating.attr('task'); | |||
if (typeof task === 'undefined') { | |||
@@ -223,9 +224,13 @@ function initRepoStatusChecker() { | |||
$('#repo_migrating').hide(); | |||
$('#repo_migrating_failed').show(); | |||
$('#repo_migrating_failed_image').show(); | |||
$('#repo_migrating_failed_error').text(xhr.responseJSON.err); | |||
$('#repo_migrating_failed_error').text(xhr.responseJSON.message); | |||
return; | |||
} | |||
if (xhr.responseJSON.message) { | |||
$('#repo_migrating_progress_message').show(); | |||
$('#repo_migrating_progress_message').text(xhr.responseJSON.message); | |||
} | |||
setTimeout(() => { | |||
initRepoStatusChecker(); | |||
}, 2000); |