summaryrefslogtreecommitdiffstats
path: root/modules
diff options
context:
space:
mode:
authorLunny Xiao <xiaolunwen@gmail.com>2019-06-29 21:38:22 +0800
committerGitHub <noreply@github.com>2019-06-29 21:38:22 +0800
commit462284e2f5768cf04d71c7abd8c01eef20cff73d (patch)
tree10d970a11d6c604cbe99cbd1e89c6dda983d6ac7 /modules
parente463bdaf8d995d0b399c1848a780ac5aa8798f0d (diff)
downloadgitea-462284e2f5768cf04d71c7abd8c01eef20cff73d.tar.gz
gitea-462284e2f5768cf04d71c7abd8c01eef20cff73d.zip
Use batch insert on migrating repository to make the process faster (#7050)
* Use batch insert on migrating repository to make the process faster * fix lint * fix tests * fix comments
Diffstat (limited to 'modules')
-rw-r--r--modules/migrations/base/comment.go1
-rw-r--r--modules/migrations/base/uploader.go14
-rw-r--r--modules/migrations/gitea.go351
-rw-r--r--modules/migrations/github.go1
-rw-r--r--modules/migrations/github_test.go3
-rw-r--r--modules/migrations/migrate.go76
6 files changed, 259 insertions, 187 deletions
diff --git a/modules/migrations/base/comment.go b/modules/migrations/base/comment.go
index 0ff0963f07..d89ec3a3f5 100644
--- a/modules/migrations/base/comment.go
+++ b/modules/migrations/base/comment.go
@@ -9,6 +9,7 @@ import "time"
// Comment is a standard comment information
type Comment struct {
+ IssueIndex int64
PosterName string
PosterEmail string
Created time.Time
diff --git a/modules/migrations/base/uploader.go b/modules/migrations/base/uploader.go
index eaeb10314a..096a8ab8f0 100644
--- a/modules/migrations/base/uploader.go
+++ b/modules/migrations/base/uploader.go
@@ -5,14 +5,14 @@
package base
-// Uploader uploads all the informations
+// Uploader uploads all the informations of one repository
type Uploader interface {
CreateRepo(repo *Repository, includeWiki bool) error
- CreateMilestone(milestone *Milestone) error
- CreateRelease(release *Release) error
- CreateLabel(label *Label) error
- CreateIssue(issue *Issue) error
- CreateComment(issueNumber int64, comment *Comment) error
- CreatePullRequest(pr *PullRequest) error
+ CreateMilestones(milestones ...*Milestone) error
+ CreateReleases(releases ...*Release) error
+ CreateLabels(labels ...*Label) error
+ CreateIssues(issues ...*Issue) error
+ CreateComments(comments ...*Comment) error
+ CreatePullRequests(prs ...*PullRequest) error
Rollback() error
}
diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go
index 4e930fa831..7d5a50d307 100644
--- a/modules/migrations/gitea.go
+++ b/modules/migrations/gitea.go
@@ -76,238 +76,280 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, includeWiki bool)
return err
}
-// CreateMilestone creates milestone
-func (g *GiteaLocalUploader) CreateMilestone(milestone *base.Milestone) error {
- var deadline util.TimeStamp
- if milestone.Deadline != nil {
- deadline = util.TimeStamp(milestone.Deadline.Unix())
- }
- if deadline == 0 {
- deadline = util.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.UILocation).Unix())
- }
- var ms = models.Milestone{
- RepoID: g.repo.ID,
- Name: milestone.Title,
- Content: milestone.Description,
- IsClosed: milestone.State == "close",
- DeadlineUnix: deadline,
- }
- if ms.IsClosed && milestone.Closed != nil {
- ms.ClosedDateUnix = util.TimeStamp(milestone.Closed.Unix())
+// CreateMilestones creates milestones
+func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error {
+ var mss = make([]*models.Milestone, 0, len(milestones))
+ for _, milestone := range milestones {
+ var deadline util.TimeStamp
+ if milestone.Deadline != nil {
+ deadline = util.TimeStamp(milestone.Deadline.Unix())
+ }
+ if deadline == 0 {
+ deadline = util.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.UILocation).Unix())
+ }
+ var ms = models.Milestone{
+ RepoID: g.repo.ID,
+ Name: milestone.Title,
+ Content: milestone.Description,
+ IsClosed: milestone.State == "close",
+ DeadlineUnix: deadline,
+ }
+ if ms.IsClosed && milestone.Closed != nil {
+ ms.ClosedDateUnix = util.TimeStamp(milestone.Closed.Unix())
+ }
+ mss = append(mss, &ms)
}
- err := models.NewMilestone(&ms)
+ err := models.InsertMilestones(mss...)
if err != nil {
return err
}
- g.milestones.Store(ms.Name, ms.ID)
- return nil
-}
-// CreateLabel creates label
-func (g *GiteaLocalUploader) CreateLabel(label *base.Label) error {
- var lb = models.Label{
- RepoID: g.repo.ID,
- Name: label.Name,
- Description: label.Description,
- Color: fmt.Sprintf("#%s", label.Color),
+ for _, ms := range mss {
+ g.milestones.Store(ms.Name, ms.ID)
}
- err := models.NewLabel(&lb)
- if err != nil {
- return err
- }
- g.labels.Store(lb.Name, lb.ID)
return nil
}
-// CreateRelease creates release
-func (g *GiteaLocalUploader) CreateRelease(release *base.Release) error {
- var rel = models.Release{
- RepoID: g.repo.ID,
- PublisherID: g.doer.ID,
- TagName: release.TagName,
- LowerTagName: strings.ToLower(release.TagName),
- Target: release.TargetCommitish,
- Title: release.Name,
- Sha1: release.TargetCommitish,
- Note: release.Body,
- IsDraft: release.Draft,
- IsPrerelease: release.Prerelease,
- IsTag: false,
- CreatedUnix: util.TimeStamp(release.Created.Unix()),
+// CreateLabels creates labels
+func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
+ var lbs = make([]*models.Label, 0, len(labels))
+ for _, label := range labels {
+ lbs = append(lbs, &models.Label{
+ RepoID: g.repo.ID,
+ Name: label.Name,
+ Description: label.Description,
+ Color: fmt.Sprintf("#%s", label.Color),
+ })
}
- // calc NumCommits
- commit, err := g.gitRepo.GetCommit(rel.TagName)
+ err := models.NewLabels(lbs...)
if err != nil {
- return fmt.Errorf("GetCommit: %v", err)
+ return err
}
- rel.NumCommits, err = commit.CommitsCount()
- if err != nil {
- return fmt.Errorf("CommitsCount: %v", err)
+ for _, lb := range lbs {
+ g.labels.Store(lb.Name, lb)
}
+ return nil
+}
- for _, asset := range release.Assets {
- var attach = models.Attachment{
- UUID: gouuid.NewV4().String(),
- Name: asset.Name,
- DownloadCount: int64(*asset.DownloadCount),
- Size: int64(*asset.Size),
- CreatedUnix: util.TimeStamp(asset.Created.Unix()),
+// CreateReleases creates releases
+func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
+ var rels = make([]*models.Release, 0, len(releases))
+ for _, release := range releases {
+ var rel = models.Release{
+ RepoID: g.repo.ID,
+ PublisherID: g.doer.ID,
+ TagName: release.TagName,
+ LowerTagName: strings.ToLower(release.TagName),
+ Target: release.TargetCommitish,
+ Title: release.Name,
+ Sha1: release.TargetCommitish,
+ Note: release.Body,
+ IsDraft: release.Draft,
+ IsPrerelease: release.Prerelease,
+ IsTag: false,
+ CreatedUnix: util.TimeStamp(release.Created.Unix()),
}
- // download attachment
- resp, err := http.Get(asset.URL)
+ // calc NumCommits
+ commit, err := g.gitRepo.GetCommit(rel.TagName)
if err != nil {
- return err
+ return fmt.Errorf("GetCommit: %v", err)
}
- defer resp.Body.Close()
-
- localPath := attach.LocalPath()
- if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
- return fmt.Errorf("MkdirAll: %v", err)
- }
-
- fw, err := os.Create(localPath)
+ rel.NumCommits, err = commit.CommitsCount()
if err != nil {
- return fmt.Errorf("Create: %v", err)
+ return fmt.Errorf("CommitsCount: %v", err)
}
- defer fw.Close()
- if _, err := io.Copy(fw, resp.Body); err != nil {
- return err
+ for _, asset := range release.Assets {
+ var attach = models.Attachment{
+ UUID: gouuid.NewV4().String(),
+ Name: asset.Name,
+ DownloadCount: int64(*asset.DownloadCount),
+ Size: int64(*asset.Size),
+ CreatedUnix: util.TimeStamp(asset.Created.Unix()),
+ }
+
+ // download attachment
+ resp, err := http.Get(asset.URL)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+
+ localPath := attach.LocalPath()
+ if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
+ return fmt.Errorf("MkdirAll: %v", err)
+ }
+
+ fw, err := os.Create(localPath)
+ if err != nil {
+ return fmt.Errorf("Create: %v", err)
+ }
+ defer fw.Close()
+
+ if _, err := io.Copy(fw, resp.Body); err != nil {
+ return err
+ }
+
+ rel.Attachments = append(rel.Attachments, &attach)
}
- rel.Attachments = append(rel.Attachments, &attach)
+ rels = append(rels, &rel)
}
-
- return models.MigrateRelease(&rel)
+ return models.InsertReleases(rels...)
}
-// CreateIssue creates issue
-func (g *GiteaLocalUploader) CreateIssue(issue *base.Issue) error {
- var labelIDs []int64
- for _, label := range issue.Labels {
- id, ok := g.labels.Load(label.Name)
- if !ok {
- return fmt.Errorf("Label %s missing when create issue", label.Name)
+// CreateIssues creates issues
+func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
+ var iss = make([]*models.Issue, 0, len(issues))
+ for _, issue := range issues {
+ var labels []*models.Label
+ for _, label := range issue.Labels {
+ lb, ok := g.labels.Load(label.Name)
+ if ok {
+ labels = append(labels, lb.(*models.Label))
+ }
}
- labelIDs = append(labelIDs, id.(int64))
- }
- var milestoneID int64
- if issue.Milestone != "" {
- milestone, ok := g.milestones.Load(issue.Milestone)
- if !ok {
- return fmt.Errorf("Milestone %s missing when create issue", issue.Milestone)
+ var milestoneID int64
+ if issue.Milestone != "" {
+ milestone, ok := g.milestones.Load(issue.Milestone)
+ if ok {
+ milestoneID = milestone.(int64)
+ }
}
- milestoneID = milestone.(int64)
- }
- var is = models.Issue{
- RepoID: g.repo.ID,
- Repo: g.repo,
- Index: issue.Number,
- PosterID: g.doer.ID,
- Title: issue.Title,
- Content: issue.Content,
- IsClosed: issue.State == "closed",
- IsLocked: issue.IsLocked,
- MilestoneID: milestoneID,
- CreatedUnix: util.TimeStamp(issue.Created.Unix()),
- }
- if issue.Closed != nil {
- is.ClosedUnix = util.TimeStamp(issue.Closed.Unix())
+ var is = models.Issue{
+ RepoID: g.repo.ID,
+ Repo: g.repo,
+ Index: issue.Number,
+ PosterID: g.doer.ID,
+ Title: issue.Title,
+ Content: issue.Content,
+ IsClosed: issue.State == "closed",
+ IsLocked: issue.IsLocked,
+ MilestoneID: milestoneID,
+ Labels: labels,
+ CreatedUnix: util.TimeStamp(issue.Created.Unix()),
+ }
+ if issue.Closed != nil {
+ is.ClosedUnix = util.TimeStamp(issue.Closed.Unix())
+ }
+ // TODO: add reactions
+ iss = append(iss, &is)
}
- err := models.InsertIssue(&is, labelIDs)
+ err := models.InsertIssues(iss...)
if err != nil {
return err
}
- g.issues.Store(issue.Number, is.ID)
- // TODO: add reactions
- return err
+ for _, is := range iss {
+ g.issues.Store(is.Index, is.ID)
+ }
+ return nil
+}
+
+// CreateComments creates comments of issues
+func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
+ var cms = make([]*models.Comment, 0, len(comments))
+ for _, comment := range comments {
+ var issueID int64
+ if issueIDStr, ok := g.issues.Load(comment.IssueIndex); !ok {
+ issue, err := models.GetIssueByIndex(g.repo.ID, comment.IssueIndex)
+ if err != nil {
+ return err
+ }
+ issueID = issue.ID
+ g.issues.Store(comment.IssueIndex, issueID)
+ } else {
+ issueID = issueIDStr.(int64)
+ }
+
+ cms = append(cms, &models.Comment{
+ IssueID: issueID,
+ Type: models.CommentTypeComment,
+ PosterID: g.doer.ID,
+ Content: comment.Content,
+ CreatedUnix: util.TimeStamp(comment.Created.Unix()),
+ })
+
+ // TODO: Reactions
+ }
+
+ return models.InsertIssueComments(cms)
}
-// CreateComment creates comment
-func (g *GiteaLocalUploader) CreateComment(issueNumber int64, comment *base.Comment) error {
- var issueID int64
- if issueIDStr, ok := g.issues.Load(issueNumber); !ok {
- issue, err := models.GetIssueByIndex(g.repo.ID, issueNumber)
+// CreatePullRequests creates pull requests
+func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error {
+ var gprs = make([]*models.PullRequest, 0, len(prs))
+ for _, pr := range prs {
+ gpr, err := g.newPullRequest(pr)
if err != nil {
return err
}
- issueID = issue.ID
- g.issues.Store(issueNumber, issueID)
- } else {
- issueID = issueIDStr.(int64)
+ gprs = append(gprs, gpr)
}
-
- var cm = models.Comment{
- IssueID: issueID,
- Type: models.CommentTypeComment,
- PosterID: g.doer.ID,
- Content: comment.Content,
- CreatedUnix: util.TimeStamp(comment.Created.Unix()),
+ if err := models.InsertPullRequests(gprs...); err != nil {
+ return err
}
- err := models.InsertComment(&cm)
- // TODO: Reactions
- return err
+ for _, pr := range gprs {
+ g.issues.Store(pr.Issue.Index, pr.Issue.ID)
+ }
+ return nil
}
-// CreatePullRequest creates pull request
-func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error {
- var labelIDs []int64
+func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) {
+ var labels []*models.Label
for _, label := range pr.Labels {
- id, ok := g.labels.Load(label.Name)
- if !ok {
- return fmt.Errorf("Label %s missing when create issue", label.Name)
+ lb, ok := g.labels.Load(label.Name)
+ if ok {
+ labels = append(labels, lb.(*models.Label))
}
- labelIDs = append(labelIDs, id.(int64))
}
var milestoneID int64
if pr.Milestone != "" {
milestone, ok := g.milestones.Load(pr.Milestone)
- if !ok {
- return fmt.Errorf("Milestone %s missing when create issue", pr.Milestone)
+ if ok {
+ milestoneID = milestone.(int64)
}
- milestoneID = milestone.(int64)
}
// download patch file
resp, err := http.Get(pr.PatchURL)
if err != nil {
- return err
+ return nil, err
}
defer resp.Body.Close()
pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
- return err
+ return nil, err
}
f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)))
if err != nil {
- return err
+ return nil, err
}
defer f.Close()
_, err = io.Copy(f, resp.Body)
if err != nil {
- return err
+ return nil, err
}
// set head information
pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
- return err
+ return nil, err
}
p, err := os.Create(filepath.Join(pullHead, "head"))
if err != nil {
- return err
+ return nil, err
}
defer p.Close()
_, err = p.WriteString(pr.Head.SHA)
if err != nil {
- return err
+ return nil, err
}
var head = "unknown repository"
@@ -333,16 +375,16 @@ func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error {
} else {
headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
- return err
+ return nil, err
}
b, err := os.Create(headBranch)
if err != nil {
- return err
+ return nil, err
}
defer b.Close()
_, err = b.WriteString(pr.Head.SHA)
if err != nil {
- return err
+ return nil, err
}
head = pr.Head.OwnerName + "/" + pr.Head.Ref
}
@@ -373,6 +415,7 @@ func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error {
IsPull: true,
IsClosed: pr.State == "closed",
IsLocked: pr.IsLocked,
+ Labels: labels,
CreatedUnix: util.TimeStamp(pr.Created.Unix()),
},
}
@@ -389,7 +432,7 @@ func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error {
// TODO: reactions
// TODO: assignees
- return models.InsertPullRequest(&pullRequest, labelIDs)
+ return &pullRequest, nil
}
// Rollback when migrating failed, this will rollback all the changes.
diff --git a/modules/migrations/github.go b/modules/migrations/github.go
index 6847787ace..e6b532df9c 100644
--- a/modules/migrations/github.go
+++ b/modules/migrations/github.go
@@ -358,6 +358,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
reactions = convertGithubReactions(comment.Reactions)
}
allComments = append(allComments, &base.Comment{
+ IssueIndex: issueNumber,
PosterName: *comment.User.Login,
PosterEmail: email,
Content: *comment.Body,
diff --git a/modules/migrations/github_test.go b/modules/migrations/github_test.go
index c14292ecc6..700183bdc1 100644
--- a/modules/migrations/github_test.go
+++ b/modules/migrations/github_test.go
@@ -269,6 +269,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
assert.EqualValues(t, 35, len(comments))
assert.EqualValues(t, []*base.Comment{
{
+ IssueIndex: 6,
PosterName: "bkcsoft",
Created: time.Date(2016, 11, 02, 18, 59, 48, 0, time.UTC),
Content: `I would prefer a solution that is in the backend, unless it's required to have it update without reloading. Unfortunately I can't seem to find anything that does that :unamused:
@@ -286,6 +287,7 @@ Also this would _require_ caching, since it will fetch huge amounts of data from
},
},
{
+ IssueIndex: 6,
PosterName: "joubertredrat",
Created: time.Date(2016, 11, 02, 19, 16, 56, 0, time.UTC),
Content: `Yes, this plugin build on front-end, with backend I don't know too, but we can consider make component for this.
@@ -303,6 +305,7 @@ In my case I use ajax to get data, but build on frontend anyway
},
},
{
+ IssueIndex: 6,
PosterName: "xinity",
Created: time.Date(2016, 11, 03, 13, 04, 56, 0, time.UTC),
Content: `following @bkcsoft retention strategy in cache is a must if we don't want gitea to waste ressources.
diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go
index 4b1229f949..3a4e6b2dbe 100644
--- a/modules/migrations/migrate.go
+++ b/modules/migrations/migrate.go
@@ -91,10 +91,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
return err
}
- for _, milestone := range milestones {
- if err := uploader.CreateMilestone(milestone); err != nil {
- return err
- }
+ if err := uploader.CreateMilestones(milestones...); err != nil {
+ return err
}
}
@@ -105,10 +103,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
return err
}
- for _, label := range labels {
- if err := uploader.CreateLabel(label); err != nil {
- return err
- }
+ if err := uploader.CreateLabels(labels...); err != nil {
+ return err
}
}
@@ -119,10 +115,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
return err
}
- for _, release := range releases {
- if err := uploader.CreateRelease(release); err != nil {
- return err
- }
+ if err := uploader.CreateReleases(releases...); err != nil {
+ return err
}
}
@@ -137,15 +131,18 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if !opts.IgnoreIssueAuthor {
issue.Content = fmt.Sprintf("Author: @%s \n\n%s", issue.PosterName, issue.Content)
}
+ }
- if err := uploader.CreateIssue(issue); err != nil {
- return err
- }
+ if err := uploader.CreateIssues(issues...); err != nil {
+ return err
+ }
- if !opts.Comments {
- continue
- }
+ if !opts.Comments {
+ continue
+ }
+ var allComments = make([]*base.Comment, 0, 100)
+ for _, issue := range issues {
comments, err := downloader.GetComments(issue.Number)
if err != nil {
return err
@@ -154,9 +151,20 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if !opts.IgnoreIssueAuthor {
comment.Content = fmt.Sprintf("Author: @%s \n\n%s", comment.PosterName, comment.Content)
}
- if err := uploader.CreateComment(issue.Number, comment); err != nil {
+ }
+ allComments = append(allComments, comments...)
+
+ if len(allComments) >= 100 {
+ if err := uploader.CreateComments(allComments...); err != nil {
return err
}
+ allComments = make([]*base.Comment, 0, 100)
+ }
+ }
+
+ if len(allComments) > 0 {
+ if err := uploader.CreateComments(allComments...); err != nil {
+ return err
}
}
@@ -178,13 +186,17 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if !opts.IgnoreIssueAuthor {
pr.Content = fmt.Sprintf("Author: @%s \n\n%s", pr.PosterName, pr.Content)
}
- if err := uploader.CreatePullRequest(pr); err != nil {
- return err
- }
- if !opts.Comments {
- continue
- }
+ }
+ if err := uploader.CreatePullRequests(prs...); err != nil {
+ return err
+ }
+ if !opts.Comments {
+ continue
+ }
+
+ var allComments = make([]*base.Comment, 0, 100)
+ for _, pr := range prs {
comments, err := downloader.GetComments(pr.Number)
if err != nil {
return err
@@ -193,11 +205,23 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if !opts.IgnoreIssueAuthor {
comment.Content = fmt.Sprintf("Author: @%s \n\n%s", comment.PosterName, comment.Content)
}
- if err := uploader.CreateComment(pr.Number, comment); err != nil {
+ }
+
+ allComments = append(allComments, comments...)
+
+ if len(allComments) >= 100 {
+ if err := uploader.CreateComments(allComments...); err != nil {
return err
}
+ allComments = make([]*base.Comment, 0, 100)
}
}
+ if len(allComments) > 0 {
+ if err := uploader.CreateComments(allComments...); err != nil {
+ return err
+ }
+ }
+
if len(prs) < 100 {
break
}