From 09663493543a2ce15fc90fbb3808dda5b87335e5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 30 Jun 2021 15:23:49 +0800 Subject: Make the github migration less rate limit waiting to get comment per page from repository but not per issue (#16070) * Make the github migration less rate limit waiting to get comment per page from repository but not per issue * Fix lint * adjust Downloader interface * Fix missed reviews * Fix test * Remove unused struct --- modules/migrations/base/downloader.go | 10 +++++++++- modules/migrations/base/null_downloader.go | 9 +++++++-- modules/migrations/base/retry_downloader.go | 7 ++++--- 3 files changed, 20 insertions(+), 6 deletions(-) (limited to 'modules/migrations/base') diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 919f4b52a0..2388b2dd6e 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -11,6 +11,13 @@ import ( "code.gitea.io/gitea/modules/structs" ) +// GetCommentOptions represents an options for get comment +type GetCommentOptions struct { + IssueNumber int64 + Page int + PageSize int +} + // Downloader downloads the site repo informations type Downloader interface { SetContext(context.Context) @@ -20,7 +27,8 @@ type Downloader interface { GetReleases() ([]*Release, error) GetLabels() ([]*Label, error) GetIssues(page, perPage int) ([]*Issue, bool, error) - GetComments(issueNumber int64) ([]*Comment, error) + GetComments(opts GetCommentOptions) ([]*Comment, bool, error) + SupportGetRepoComments() bool GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) GetReviews(pullRequestNumber int64) ([]*Review, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) diff --git a/modules/migrations/base/null_downloader.go b/modules/migrations/base/null_downloader.go index a93c20339b..53a536709d 100644 --- a/modules/migrations/base/null_downloader.go +++ b/modules/migrations/base/null_downloader.go @@ -51,8 +51,8 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns comments according issueNumber -func (n NullDownloader) GetComments(issueNumber int64) ([]*Comment, error) { - return nil, &ErrNotSupported{Entity: "Comments"} +func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) { + return nil, false, &ErrNotSupported{Entity: "Comments"} } // GetPullRequests returns pull requests according page and perPage @@ -80,3 +80,8 @@ func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) ( } return remoteAddr, nil } + +// SupportGetRepoComments return true if it supports get repo comments +func (n NullDownloader) SupportGetRepoComments() bool { + return false +} diff --git a/modules/migrations/base/retry_downloader.go b/modules/migrations/base/retry_downloader.go index 82a038b98b..e6c80038f1 100644 --- a/modules/migrations/base/retry_downloader.go +++ b/modules/migrations/base/retry_downloader.go @@ -150,18 +150,19 @@ func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns a repository's comments with retry -func (d *RetryDownloader) GetComments(issueNumber int64) ([]*Comment, error) { +func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) { var ( comments []*Comment + isEnd bool err error ) err = d.retry(func() error { - comments, err = d.Downloader.GetComments(issueNumber) + comments, isEnd, err = d.Downloader.GetComments(opts) return err }) - return comments, err + return comments, isEnd, err } // GetPullRequests returns a repository's pull requests with retry -- cgit v1.2.3