diff options
author | Lunny Xiao <xiaolunwen@gmail.com> | 2019-06-29 21:38:22 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-29 21:38:22 +0800 |
commit | 462284e2f5768cf04d71c7abd8c01eef20cff73d (patch) | |
tree | 10d970a11d6c604cbe99cbd1e89c6dda983d6ac7 /modules | |
parent | e463bdaf8d995d0b399c1848a780ac5aa8798f0d (diff) | |
download | gitea-462284e2f5768cf04d71c7abd8c01eef20cff73d.tar.gz gitea-462284e2f5768cf04d71c7abd8c01eef20cff73d.zip |
Use batch insert on migrating repository to make the process faster (#7050)
* Use batch insert on migrating repository to make the process faster
* fix lint
* fix tests
* fix comments
Diffstat (limited to 'modules')
-rw-r--r-- | modules/migrations/base/comment.go | 1 | ||||
-rw-r--r-- | modules/migrations/base/uploader.go | 14 | ||||
-rw-r--r-- | modules/migrations/gitea.go | 351 | ||||
-rw-r--r-- | modules/migrations/github.go | 1 | ||||
-rw-r--r-- | modules/migrations/github_test.go | 3 | ||||
-rw-r--r-- | modules/migrations/migrate.go | 76 |
6 files changed, 259 insertions, 187 deletions
diff --git a/modules/migrations/base/comment.go b/modules/migrations/base/comment.go index 0ff0963f07..d89ec3a3f5 100644 --- a/modules/migrations/base/comment.go +++ b/modules/migrations/base/comment.go @@ -9,6 +9,7 @@ import "time" // Comment is a standard comment information type Comment struct { + IssueIndex int64 PosterName string PosterEmail string Created time.Time diff --git a/modules/migrations/base/uploader.go b/modules/migrations/base/uploader.go index eaeb10314a..096a8ab8f0 100644 --- a/modules/migrations/base/uploader.go +++ b/modules/migrations/base/uploader.go @@ -5,14 +5,14 @@ package base -// Uploader uploads all the informations +// Uploader uploads all the informations of one repository type Uploader interface { CreateRepo(repo *Repository, includeWiki bool) error - CreateMilestone(milestone *Milestone) error - CreateRelease(release *Release) error - CreateLabel(label *Label) error - CreateIssue(issue *Issue) error - CreateComment(issueNumber int64, comment *Comment) error - CreatePullRequest(pr *PullRequest) error + CreateMilestones(milestones ...*Milestone) error + CreateReleases(releases ...*Release) error + CreateLabels(labels ...*Label) error + CreateIssues(issues ...*Issue) error + CreateComments(comments ...*Comment) error + CreatePullRequests(prs ...*PullRequest) error Rollback() error } diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index 4e930fa831..7d5a50d307 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -76,238 +76,280 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, includeWiki bool) return err } -// CreateMilestone creates milestone -func (g *GiteaLocalUploader) CreateMilestone(milestone *base.Milestone) error { - var deadline util.TimeStamp - if milestone.Deadline != nil { - deadline = util.TimeStamp(milestone.Deadline.Unix()) - } - if deadline == 0 { - deadline = util.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.UILocation).Unix()) - } - var ms = models.Milestone{ - RepoID: g.repo.ID, - Name: milestone.Title, - Content: milestone.Description, - IsClosed: milestone.State == "close", - DeadlineUnix: deadline, - } - if ms.IsClosed && milestone.Closed != nil { - ms.ClosedDateUnix = util.TimeStamp(milestone.Closed.Unix()) +// CreateMilestones creates milestones +func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error { + var mss = make([]*models.Milestone, 0, len(milestones)) + for _, milestone := range milestones { + var deadline util.TimeStamp + if milestone.Deadline != nil { + deadline = util.TimeStamp(milestone.Deadline.Unix()) + } + if deadline == 0 { + deadline = util.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.UILocation).Unix()) + } + var ms = models.Milestone{ + RepoID: g.repo.ID, + Name: milestone.Title, + Content: milestone.Description, + IsClosed: milestone.State == "close", + DeadlineUnix: deadline, + } + if ms.IsClosed && milestone.Closed != nil { + ms.ClosedDateUnix = util.TimeStamp(milestone.Closed.Unix()) + } + mss = append(mss, &ms) } - err := models.NewMilestone(&ms) + err := models.InsertMilestones(mss...) if err != nil { return err } - g.milestones.Store(ms.Name, ms.ID) - return nil -} -// CreateLabel creates label -func (g *GiteaLocalUploader) CreateLabel(label *base.Label) error { - var lb = models.Label{ - RepoID: g.repo.ID, - Name: label.Name, - Description: label.Description, - Color: fmt.Sprintf("#%s", label.Color), + for _, ms := range mss { + g.milestones.Store(ms.Name, ms.ID) } - err := models.NewLabel(&lb) - if err != nil { - return err - } - g.labels.Store(lb.Name, lb.ID) return nil } -// CreateRelease creates release -func (g *GiteaLocalUploader) CreateRelease(release *base.Release) error { - var rel = models.Release{ - RepoID: g.repo.ID, - PublisherID: g.doer.ID, - TagName: release.TagName, - LowerTagName: strings.ToLower(release.TagName), - Target: release.TargetCommitish, - Title: release.Name, - Sha1: release.TargetCommitish, - Note: release.Body, - IsDraft: release.Draft, - IsPrerelease: release.Prerelease, - IsTag: false, - CreatedUnix: util.TimeStamp(release.Created.Unix()), +// CreateLabels creates labels +func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { + var lbs = make([]*models.Label, 0, len(labels)) + for _, label := range labels { + lbs = append(lbs, &models.Label{ + RepoID: g.repo.ID, + Name: label.Name, + Description: label.Description, + Color: fmt.Sprintf("#%s", label.Color), + }) } - // calc NumCommits - commit, err := g.gitRepo.GetCommit(rel.TagName) + err := models.NewLabels(lbs...) if err != nil { - return fmt.Errorf("GetCommit: %v", err) + return err } - rel.NumCommits, err = commit.CommitsCount() - if err != nil { - return fmt.Errorf("CommitsCount: %v", err) + for _, lb := range lbs { + g.labels.Store(lb.Name, lb) } + return nil +} - for _, asset := range release.Assets { - var attach = models.Attachment{ - UUID: gouuid.NewV4().String(), - Name: asset.Name, - DownloadCount: int64(*asset.DownloadCount), - Size: int64(*asset.Size), - CreatedUnix: util.TimeStamp(asset.Created.Unix()), +// CreateReleases creates releases +func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { + var rels = make([]*models.Release, 0, len(releases)) + for _, release := range releases { + var rel = models.Release{ + RepoID: g.repo.ID, + PublisherID: g.doer.ID, + TagName: release.TagName, + LowerTagName: strings.ToLower(release.TagName), + Target: release.TargetCommitish, + Title: release.Name, + Sha1: release.TargetCommitish, + Note: release.Body, + IsDraft: release.Draft, + IsPrerelease: release.Prerelease, + IsTag: false, + CreatedUnix: util.TimeStamp(release.Created.Unix()), } - // download attachment - resp, err := http.Get(asset.URL) + // calc NumCommits + commit, err := g.gitRepo.GetCommit(rel.TagName) if err != nil { - return err + return fmt.Errorf("GetCommit: %v", err) } - defer resp.Body.Close() - - localPath := attach.LocalPath() - if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil { - return fmt.Errorf("MkdirAll: %v", err) - } - - fw, err := os.Create(localPath) + rel.NumCommits, err = commit.CommitsCount() if err != nil { - return fmt.Errorf("Create: %v", err) + return fmt.Errorf("CommitsCount: %v", err) } - defer fw.Close() - if _, err := io.Copy(fw, resp.Body); err != nil { - return err + for _, asset := range release.Assets { + var attach = models.Attachment{ + UUID: gouuid.NewV4().String(), + Name: asset.Name, + DownloadCount: int64(*asset.DownloadCount), + Size: int64(*asset.Size), + CreatedUnix: util.TimeStamp(asset.Created.Unix()), + } + + // download attachment + resp, err := http.Get(asset.URL) + if err != nil { + return err + } + defer resp.Body.Close() + + localPath := attach.LocalPath() + if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil { + return fmt.Errorf("MkdirAll: %v", err) + } + + fw, err := os.Create(localPath) + if err != nil { + return fmt.Errorf("Create: %v", err) + } + defer fw.Close() + + if _, err := io.Copy(fw, resp.Body); err != nil { + return err + } + + rel.Attachments = append(rel.Attachments, &attach) } - rel.Attachments = append(rel.Attachments, &attach) + rels = append(rels, &rel) } - - return models.MigrateRelease(&rel) + return models.InsertReleases(rels...) } -// CreateIssue creates issue -func (g *GiteaLocalUploader) CreateIssue(issue *base.Issue) error { - var labelIDs []int64 - for _, label := range issue.Labels { - id, ok := g.labels.Load(label.Name) - if !ok { - return fmt.Errorf("Label %s missing when create issue", label.Name) +// CreateIssues creates issues +func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { + var iss = make([]*models.Issue, 0, len(issues)) + for _, issue := range issues { + var labels []*models.Label + for _, label := range issue.Labels { + lb, ok := g.labels.Load(label.Name) + if ok { + labels = append(labels, lb.(*models.Label)) + } } - labelIDs = append(labelIDs, id.(int64)) - } - var milestoneID int64 - if issue.Milestone != "" { - milestone, ok := g.milestones.Load(issue.Milestone) - if !ok { - return fmt.Errorf("Milestone %s missing when create issue", issue.Milestone) + var milestoneID int64 + if issue.Milestone != "" { + milestone, ok := g.milestones.Load(issue.Milestone) + if ok { + milestoneID = milestone.(int64) + } } - milestoneID = milestone.(int64) - } - var is = models.Issue{ - RepoID: g.repo.ID, - Repo: g.repo, - Index: issue.Number, - PosterID: g.doer.ID, - Title: issue.Title, - Content: issue.Content, - IsClosed: issue.State == "closed", - IsLocked: issue.IsLocked, - MilestoneID: milestoneID, - CreatedUnix: util.TimeStamp(issue.Created.Unix()), - } - if issue.Closed != nil { - is.ClosedUnix = util.TimeStamp(issue.Closed.Unix()) + var is = models.Issue{ + RepoID: g.repo.ID, + Repo: g.repo, + Index: issue.Number, + PosterID: g.doer.ID, + Title: issue.Title, + Content: issue.Content, + IsClosed: issue.State == "closed", + IsLocked: issue.IsLocked, + MilestoneID: milestoneID, + Labels: labels, + CreatedUnix: util.TimeStamp(issue.Created.Unix()), + } + if issue.Closed != nil { + is.ClosedUnix = util.TimeStamp(issue.Closed.Unix()) + } + // TODO: add reactions + iss = append(iss, &is) } - err := models.InsertIssue(&is, labelIDs) + err := models.InsertIssues(iss...) if err != nil { return err } - g.issues.Store(issue.Number, is.ID) - // TODO: add reactions - return err + for _, is := range iss { + g.issues.Store(is.Index, is.ID) + } + return nil +} + +// CreateComments creates comments of issues +func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { + var cms = make([]*models.Comment, 0, len(comments)) + for _, comment := range comments { + var issueID int64 + if issueIDStr, ok := g.issues.Load(comment.IssueIndex); !ok { + issue, err := models.GetIssueByIndex(g.repo.ID, comment.IssueIndex) + if err != nil { + return err + } + issueID = issue.ID + g.issues.Store(comment.IssueIndex, issueID) + } else { + issueID = issueIDStr.(int64) + } + + cms = append(cms, &models.Comment{ + IssueID: issueID, + Type: models.CommentTypeComment, + PosterID: g.doer.ID, + Content: comment.Content, + CreatedUnix: util.TimeStamp(comment.Created.Unix()), + }) + + // TODO: Reactions + } + + return models.InsertIssueComments(cms) } -// CreateComment creates comment -func (g *GiteaLocalUploader) CreateComment(issueNumber int64, comment *base.Comment) error { - var issueID int64 - if issueIDStr, ok := g.issues.Load(issueNumber); !ok { - issue, err := models.GetIssueByIndex(g.repo.ID, issueNumber) +// CreatePullRequests creates pull requests +func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error { + var gprs = make([]*models.PullRequest, 0, len(prs)) + for _, pr := range prs { + gpr, err := g.newPullRequest(pr) if err != nil { return err } - issueID = issue.ID - g.issues.Store(issueNumber, issueID) - } else { - issueID = issueIDStr.(int64) + gprs = append(gprs, gpr) } - - var cm = models.Comment{ - IssueID: issueID, - Type: models.CommentTypeComment, - PosterID: g.doer.ID, - Content: comment.Content, - CreatedUnix: util.TimeStamp(comment.Created.Unix()), + if err := models.InsertPullRequests(gprs...); err != nil { + return err } - err := models.InsertComment(&cm) - // TODO: Reactions - return err + for _, pr := range gprs { + g.issues.Store(pr.Issue.Index, pr.Issue.ID) + } + return nil } -// CreatePullRequest creates pull request -func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error { - var labelIDs []int64 +func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) { + var labels []*models.Label for _, label := range pr.Labels { - id, ok := g.labels.Load(label.Name) - if !ok { - return fmt.Errorf("Label %s missing when create issue", label.Name) + lb, ok := g.labels.Load(label.Name) + if ok { + labels = append(labels, lb.(*models.Label)) } - labelIDs = append(labelIDs, id.(int64)) } var milestoneID int64 if pr.Milestone != "" { milestone, ok := g.milestones.Load(pr.Milestone) - if !ok { - return fmt.Errorf("Milestone %s missing when create issue", pr.Milestone) + if ok { + milestoneID = milestone.(int64) } - milestoneID = milestone.(int64) } // download patch file resp, err := http.Get(pr.PatchURL) if err != nil { - return err + return nil, err } defer resp.Body.Close() pullDir := filepath.Join(g.repo.RepoPath(), "pulls") if err = os.MkdirAll(pullDir, os.ModePerm); err != nil { - return err + return nil, err } f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))) if err != nil { - return err + return nil, err } defer f.Close() _, err = io.Copy(f, resp.Body) if err != nil { - return err + return nil, err } // set head information pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { - return err + return nil, err } p, err := os.Create(filepath.Join(pullHead, "head")) if err != nil { - return err + return nil, err } defer p.Close() _, err = p.WriteString(pr.Head.SHA) if err != nil { - return err + return nil, err } var head = "unknown repository" @@ -333,16 +375,16 @@ func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error { } else { headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { - return err + return nil, err } b, err := os.Create(headBranch) if err != nil { - return err + return nil, err } defer b.Close() _, err = b.WriteString(pr.Head.SHA) if err != nil { - return err + return nil, err } head = pr.Head.OwnerName + "/" + pr.Head.Ref } @@ -373,6 +415,7 @@ func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error { IsPull: true, IsClosed: pr.State == "closed", IsLocked: pr.IsLocked, + Labels: labels, CreatedUnix: util.TimeStamp(pr.Created.Unix()), }, } @@ -389,7 +432,7 @@ func (g *GiteaLocalUploader) CreatePullRequest(pr *base.PullRequest) error { // TODO: reactions // TODO: assignees - return models.InsertPullRequest(&pullRequest, labelIDs) + return &pullRequest, nil } // Rollback when migrating failed, this will rollback all the changes. diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 6847787ace..e6b532df9c 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -358,6 +358,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er reactions = convertGithubReactions(comment.Reactions) } allComments = append(allComments, &base.Comment{ + IssueIndex: issueNumber, PosterName: *comment.User.Login, PosterEmail: email, Content: *comment.Body, diff --git a/modules/migrations/github_test.go b/modules/migrations/github_test.go index c14292ecc6..700183bdc1 100644 --- a/modules/migrations/github_test.go +++ b/modules/migrations/github_test.go @@ -269,6 +269,7 @@ func TestGitHubDownloadRepo(t *testing.T) { assert.EqualValues(t, 35, len(comments)) assert.EqualValues(t, []*base.Comment{ { + IssueIndex: 6, PosterName: "bkcsoft", Created: time.Date(2016, 11, 02, 18, 59, 48, 0, time.UTC), Content: `I would prefer a solution that is in the backend, unless it's required to have it update without reloading. Unfortunately I can't seem to find anything that does that :unamused: @@ -286,6 +287,7 @@ Also this would _require_ caching, since it will fetch huge amounts of data from }, }, { + IssueIndex: 6, PosterName: "joubertredrat", Created: time.Date(2016, 11, 02, 19, 16, 56, 0, time.UTC), Content: `Yes, this plugin build on front-end, with backend I don't know too, but we can consider make component for this. @@ -303,6 +305,7 @@ In my case I use ajax to get data, but build on frontend anyway }, }, { + IssueIndex: 6, PosterName: "xinity", Created: time.Date(2016, 11, 03, 13, 04, 56, 0, time.UTC), Content: `following @bkcsoft retention strategy in cache is a must if we don't want gitea to waste ressources. diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 4b1229f949..3a4e6b2dbe 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -91,10 +91,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return err } - for _, milestone := range milestones { - if err := uploader.CreateMilestone(milestone); err != nil { - return err - } + if err := uploader.CreateMilestones(milestones...); err != nil { + return err } } @@ -105,10 +103,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return err } - for _, label := range labels { - if err := uploader.CreateLabel(label); err != nil { - return err - } + if err := uploader.CreateLabels(labels...); err != nil { + return err } } @@ -119,10 +115,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return err } - for _, release := range releases { - if err := uploader.CreateRelease(release); err != nil { - return err - } + if err := uploader.CreateReleases(releases...); err != nil { + return err } } @@ -137,15 +131,18 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts if !opts.IgnoreIssueAuthor { issue.Content = fmt.Sprintf("Author: @%s \n\n%s", issue.PosterName, issue.Content) } + } - if err := uploader.CreateIssue(issue); err != nil { - return err - } + if err := uploader.CreateIssues(issues...); err != nil { + return err + } - if !opts.Comments { - continue - } + if !opts.Comments { + continue + } + var allComments = make([]*base.Comment, 0, 100) + for _, issue := range issues { comments, err := downloader.GetComments(issue.Number) if err != nil { return err @@ -154,9 +151,20 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts if !opts.IgnoreIssueAuthor { comment.Content = fmt.Sprintf("Author: @%s \n\n%s", comment.PosterName, comment.Content) } - if err := uploader.CreateComment(issue.Number, comment); err != nil { + } + allComments = append(allComments, comments...) + + if len(allComments) >= 100 { + if err := uploader.CreateComments(allComments...); err != nil { return err } + allComments = make([]*base.Comment, 0, 100) + } + } + + if len(allComments) > 0 { + if err := uploader.CreateComments(allComments...); err != nil { + return err } } @@ -178,13 +186,17 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts if !opts.IgnoreIssueAuthor { pr.Content = fmt.Sprintf("Author: @%s \n\n%s", pr.PosterName, pr.Content) } - if err := uploader.CreatePullRequest(pr); err != nil { - return err - } - if !opts.Comments { - continue - } + } + if err := uploader.CreatePullRequests(prs...); err != nil { + return err + } + if !opts.Comments { + continue + } + + var allComments = make([]*base.Comment, 0, 100) + for _, pr := range prs { comments, err := downloader.GetComments(pr.Number) if err != nil { return err @@ -193,11 +205,23 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts if !opts.IgnoreIssueAuthor { comment.Content = fmt.Sprintf("Author: @%s \n\n%s", comment.PosterName, comment.Content) } - if err := uploader.CreateComment(pr.Number, comment); err != nil { + } + + allComments = append(allComments, comments...) + + if len(allComments) >= 100 { + if err := uploader.CreateComments(allComments...); err != nil { return err } + allComments = make([]*base.Comment, 0, 100) } } + if len(allComments) > 0 { + if err := uploader.CreateComments(allComments...); err != nil { + return err + } + } + if len(prs) < 100 { break } |