Sleep longer if request speed is over github limitation (#9335)

* Sleep longer if request speed is over github limitation

* improve code

* remove unused code

* fix lint

* Use github's rate limit remain value to determine how long to sleep

* Save reset time when finished github api request

* fix bug

* fix lint

* Add context.Context for sleep

* fix test

* improve code

* fix bug and lint

* fix import order

1 files changed, 54 insertions, 10 deletions

diff --git a/modules/migrations/github.go b/modules/migrations/github.go
index 00d137a3de..fabdb4ae44 100644
--- a/modules/migrations/github.go
+++ b/modules/migrations/github.go
@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
@@ -73,6 +74,7 @@ type GithubDownloaderV3 struct {
 	repoName  string
 	userName  string
 	password  string
+	rate      *github.Rate
 }
 
 // NewGithubDownloaderV3 creates a github Downloader via github v3 API
@@ -107,12 +109,39 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith
 	return &downloader
 }
 
+// SetContext set context
+func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
+	g.ctx = ctx
+}
+
+func (g *GithubDownloaderV3) sleep() {
+	for g.rate != nil && g.rate.Remaining <= 0 {
+		timer := time.NewTimer(time.Until(g.rate.Reset.Time))
+		select {
+		case <-g.ctx.Done():
+			timer.Stop()
+			return
+		case <-timer.C:
+		}
+
+		rates, _, err := g.client.RateLimits(g.ctx)
+		if err != nil {
+			log.Error("g.client.RateLimits: %s", err)
+		}
+
+		g.rate = rates.GetCore()
+	}
+}
+
 // GetRepoInfo returns a repository information
 func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
-	gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+	g.sleep()
+	gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
 	if err != nil {
 		return nil, err
 	}
+	g.rate = &resp.Rate
+
 	// convert github repo to stand Repo
 	return &base.Repository{
 		Owner:       g.repoOwner,
@@ -126,8 +155,13 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
 
 // GetTopics return github topics
 func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
-	r, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
-	return r.Topics, err
+	g.sleep()
+	r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+	if err != nil {
+		return nil, err
+	}
+	g.rate = &resp.Rate
+	return r.Topics, nil
 }
 
 // GetMilestones returns milestones
@@ -135,7 +169,8 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
 	var perPage = 100
 	var milestones = make([]*base.Milestone, 0, perPage)
 	for i := 1; ; i++ {
-		ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
+		g.sleep()
+		ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
 			&github.MilestoneListOptions{
 				State: "all",
 				ListOptions: github.ListOptions{
@@ -145,6 +180,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
 		if err != nil {
 			return nil, err
 		}
+		g.rate = &resp.Rate
 
 		for _, m := range ms {
 			var desc string
@@ -189,7 +225,8 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
 	var perPage = 100
 	var labels = make([]*base.Label, 0, perPage)
 	for i := 1; ; i++ {
-		ls, _, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
+		g.sleep()
+		ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
 			&github.ListOptions{
 				Page:    i,
 				PerPage: perPage,
@@ -197,6 +234,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
 		if err != nil {
 			return nil, err
 		}
+		g.rate = &resp.Rate
 
 		for _, label := range ls {
 			labels = append(labels, convertGithubLabel(label))
@@ -260,7 +298,8 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
 	var perPage = 100
 	var releases = make([]*base.Release, 0, perPage)
 	for i := 1; ; i++ {
-		ls, _, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
+		g.sleep()
+		ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
 			&github.ListOptions{
 				Page:    i,
 				PerPage: perPage,
@@ -268,6 +307,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
 		if err != nil {
 			return nil, err
 		}
+		g.rate = &resp.Rate
 
 		for _, release := range ls {
 			releases = append(releases, g.convertGithubRelease(release))
@@ -304,11 +344,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
 	}
 
 	var allIssues = make([]*base.Issue, 0, perPage)
-
-	issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
+	g.sleep()
+	issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
 	if err != nil {
 		return nil, false, fmt.Errorf("error while listing repos: %v", err)
 	}
+	g.rate = &resp.Rate
 	for _, issue := range issues {
 		if issue.IsPullRequest() {
 			continue
@@ -365,10 +406,12 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
 		},
 	}
 	for {
+		g.sleep()
 		comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt)
 		if err != nil {
 			return nil, fmt.Errorf("error while listing repos: %v", err)
 		}
+		g.rate = &resp.Rate
 		for _, comment := range comments {
 			var email string
 			if comment.User.Email != nil {
@@ -408,11 +451,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
 		},
 	}
 	var allPRs = make([]*base.PullRequest, 0, perPage)
-
-	prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
+	g.sleep()
+	prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
 	if err != nil {
 		return nil, fmt.Errorf("error while listing repos: %v", err)
 	}
+	g.rate = &resp.Rate
 	for _, pr := range prs {
 		var body string
 		if pr.Body != nil {