From 08069dc4656fa53ee5dd25189e15012cb4f8acb2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 7 May 2019 09:12:51 +0800 Subject: Improve migrations to support migrating milestones/labels/issues/comments/pullrequests (#6290) * add migrations * fix package dependency * fix lints * implements migrations except pull requests * add releases * migrating releases * fix bug * fix lint * fix migrate releases * fix tests * add rollback * pull request migtations * fix import * fix go module vendor * add tests for upload to gitea * more migrate options * fix swagger-check * fix misspell * add options on migration UI * fix log error * improve UI options on migrating * add support for username password when migrating from github * fix tests * remove comments and fix migrate limitation * improve error handles * migrate API will also support migrate milestones/labels/issues/pulls/releases * fix tests and remove unused codes * add DownloaderFactory and docs about how to create a new Downloader * fix misspell * fix migration docs * Add hints about migrate options on migration page * fix tests --- models/migrate.go | 145 +++++++++++++++++++++++++++++++++++++++++++++++++ models/release_test.go | 1 + models/repo.go | 33 ++++++----- 3 files changed, 164 insertions(+), 15 deletions(-) create mode 100644 models/migrate.go (limited to 'models') diff --git a/models/migrate.go b/models/migrate.go new file mode 100644 index 0000000000..54f9a3e24c --- /dev/null +++ b/models/migrate.go @@ -0,0 +1,145 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import "github.com/go-xorm/xorm" + +// InsertIssue insert one issue to database +func InsertIssue(issue *Issue, labelIDs []int64) error { + sess := x.NewSession() + if err := sess.Begin(); err != nil { + return err + } + + if err := insertIssue(sess, issue, labelIDs); err != nil { + return err + } + return sess.Commit() +} + +func insertIssue(sess *xorm.Session, issue *Issue, labelIDs []int64) error { + if issue.MilestoneID > 0 { + sess.Incr("num_issues") + if issue.IsClosed { + sess.Incr("num_closed_issues") + } + if _, err := sess.ID(issue.MilestoneID).NoAutoTime().Update(new(Milestone)); err != nil { + return err + } + } + if _, err := sess.NoAutoTime().Insert(issue); err != nil { + return err + } + var issueLabels = make([]IssueLabel, 0, len(labelIDs)) + for _, labelID := range labelIDs { + issueLabels = append(issueLabels, IssueLabel{ + IssueID: issue.ID, + LabelID: labelID, + }) + } + if _, err := sess.Insert(issueLabels); err != nil { + return err + } + if !issue.IsPull { + sess.ID(issue.RepoID).Incr("num_issues") + if issue.IsClosed { + sess.Incr("num_closed_issues") + } + } else { + sess.ID(issue.RepoID).Incr("num_pulls") + if issue.IsClosed { + sess.Incr("num_closed_pulls") + } + } + if _, err := sess.NoAutoTime().Update(issue.Repo); err != nil { + return err + } + + sess.Incr("num_issues") + if issue.IsClosed { + sess.Incr("num_closed_issues") + } + if _, err := sess.In("id", labelIDs).Update(new(Label)); err != nil { + return err + } + + if issue.MilestoneID > 0 { + if _, err := sess.ID(issue.MilestoneID).SetExpr("completeness", "num_closed_issues * 100 / num_issues").Update(new(Milestone)); err != nil { + return err + } + } + + return nil +} + +// InsertComment inserted a comment +func InsertComment(comment *Comment) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + if _, err := sess.NoAutoTime().Insert(comment); err != nil { + return err + } + if _, err := sess.ID(comment.IssueID).Incr("num_comments").Update(new(Issue)); err != nil { + return err + } + return sess.Commit() +} + +// InsertPullRequest inserted a pull request +func InsertPullRequest(pr *PullRequest, labelIDs []int64) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + if err := insertIssue(sess, pr.Issue, labelIDs); err != nil { + return err + } + pr.IssueID = pr.Issue.ID + if _, err := sess.NoAutoTime().Insert(pr); err != nil { + return err + } + return sess.Commit() +} + +// MigrateRelease migrates release +func MigrateRelease(rel *Release) error { + sess := x.NewSession() + if err := sess.Begin(); err != nil { + return err + } + + var oriRel = Release{ + RepoID: rel.RepoID, + TagName: rel.TagName, + } + exist, err := sess.Get(&oriRel) + if err != nil { + return err + } + if !exist { + if _, err := sess.NoAutoTime().Insert(rel); err != nil { + return err + } + } else { + rel.ID = oriRel.ID + if _, err := sess.ID(rel.ID).Cols("target, title, note, is_tag, num_commits").Update(rel); err != nil { + return err + } + } + + for i := 0; i < len(rel.Attachments); i++ { + rel.Attachments[i].ReleaseID = rel.ID + } + + if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil { + return err + } + + return sess.Commit() +} diff --git a/models/release_test.go b/models/release_test.go index 39c6613054..f3f61240ea 100644 --- a/models/release_test.go +++ b/models/release_test.go @@ -107,6 +107,7 @@ func TestRelease_MirrorDelete(t *testing.T) { IsPrivate: false, IsMirror: true, RemoteAddr: repoPath, + Wiki: true, } mirror, err := MigrateRepository(user, user, migrationOptions) assert.NoError(t, err) diff --git a/models/repo.go b/models/repo.go index 936ad2ae37..e8af9aa2db 100644 --- a/models/repo.go +++ b/models/repo.go @@ -896,6 +896,7 @@ type MigrateRepoOptions struct { IsPrivate bool IsMirror bool RemoteAddr string + Wiki bool // include wiki repository } /* @@ -917,7 +918,7 @@ func wikiRemoteURL(remote string) string { return "" } -// MigrateRepository migrates a existing repository from other project hosting. +// MigrateRepository migrates an existing repository from other project hosting. func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) { repo, err := CreateRepository(doer, u, CreateRepoOptions{ Name: opts.Name, @@ -930,7 +931,6 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err } repoPath := RepoPath(u.Name, opts.Name) - wikiPath := WikiPath(u.Name, opts.Name) if u.IsOrganization() { t, err := u.GetOwnerTeam() @@ -956,22 +956,25 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err return repo, fmt.Errorf("Clone: %v", err) } - wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) - if len(wikiRemotePath) > 0 { - if err := os.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) - } - - if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ - Mirror: true, - Quiet: true, - Timeout: migrateTimeout, - Branch: "master", - }); err != nil { - log.Warn("Clone wiki: %v", err) + if opts.Wiki { + wikiPath := WikiPath(u.Name, opts.Name) + wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) + if len(wikiRemotePath) > 0 { if err := os.RemoveAll(wikiPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) } + + if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ + Mirror: true, + Quiet: true, + Timeout: migrateTimeout, + Branch: "master", + }); err != nil { + log.Warn("Clone wiki: %v", err) + if err := os.RemoveAll(wikiPath); err != nil { + return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) + } + } } } -- cgit v1.2.3