* Sleep longer if request speed is over github limitation * improve code * remove unused code * fix lint * Use github's rate limit remain value to determine how long to sleep * Save reset time when finished github api request * fix bug * fix lint * Add context.Context for sleep * fix test * improve code * fix bug and lint * fix import ordertags/v1.11.0-rc1
@@ -6,6 +6,7 @@ | |||
package base | |||
import ( | |||
"context" | |||
"time" | |||
"code.gitea.io/gitea/modules/structs" | |||
@@ -13,6 +14,7 @@ import ( | |||
// Downloader downloads the site repo informations | |||
type Downloader interface { | |||
SetContext(context.Context) | |||
GetRepoInfo() (*Repository, error) | |||
GetTopics() ([]string, error) | |||
GetMilestones() ([]*Milestone, error) | |||
@@ -30,6 +32,10 @@ type DownloaderFactory interface { | |||
GitServiceType() structs.GitServiceType | |||
} | |||
var ( | |||
_ Downloader = &RetryDownloader{} | |||
) | |||
// RetryDownloader retry the downloads | |||
type RetryDownloader struct { | |||
Downloader | |||
@@ -46,6 +52,11 @@ func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *Retr | |||
} | |||
} | |||
// SetContext set context | |||
func (d *RetryDownloader) SetContext(ctx context.Context) { | |||
d.Downloader.SetContext(ctx) | |||
} | |||
// GetRepoInfo returns a repository information with retry | |||
func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { | |||
var ( |
@@ -5,6 +5,8 @@ | |||
package migrations | |||
import ( | |||
"context" | |||
"code.gitea.io/gitea/modules/migrations/base" | |||
) | |||
@@ -28,6 +30,10 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl | |||
} | |||
} | |||
// SetContext set context | |||
func (g *PlainGitDownloader) SetContext(ctx context.Context) { | |||
} | |||
// GetRepoInfo returns a repository information | |||
func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { | |||
// convert github repo to stand Repo |
@@ -6,6 +6,7 @@ | |||
package migrations | |||
import ( | |||
"context" | |||
"fmt" | |||
"io" | |||
"net/http" | |||
@@ -35,6 +36,7 @@ var ( | |||
// GiteaLocalUploader implements an Uploader to gitea sites | |||
type GiteaLocalUploader struct { | |||
ctx context.Context | |||
doer *models.User | |||
repoOwner string | |||
repoName string | |||
@@ -49,8 +51,9 @@ type GiteaLocalUploader struct { | |||
} | |||
// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1 | |||
func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *GiteaLocalUploader { | |||
func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, repoName string) *GiteaLocalUploader { | |||
return &GiteaLocalUploader{ | |||
ctx: ctx, | |||
doer: doer, | |||
repoOwner: repoOwner, | |||
repoName: repoName, |
@@ -10,6 +10,7 @@ import ( | |||
"time" | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/graceful" | |||
"code.gitea.io/gitea/modules/structs" | |||
"code.gitea.io/gitea/modules/util" | |||
@@ -27,7 +28,7 @@ func TestGiteaUploadRepo(t *testing.T) { | |||
var ( | |||
downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder") | |||
repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05") | |||
uploader = NewGiteaLocalUploader(user, user.Name, repoName) | |||
uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) | |||
) | |||
err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{ |
@@ -11,6 +11,7 @@ import ( | |||
"net/http" | |||
"net/url" | |||
"strings" | |||
"time" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/migrations/base" | |||
@@ -73,6 +74,7 @@ type GithubDownloaderV3 struct { | |||
repoName string | |||
userName string | |||
password string | |||
rate *github.Rate | |||
} | |||
// NewGithubDownloaderV3 creates a github Downloader via github v3 API | |||
@@ -107,12 +109,39 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith | |||
return &downloader | |||
} | |||
// SetContext set context | |||
func (g *GithubDownloaderV3) SetContext(ctx context.Context) { | |||
g.ctx = ctx | |||
} | |||
func (g *GithubDownloaderV3) sleep() { | |||
for g.rate != nil && g.rate.Remaining <= 0 { | |||
timer := time.NewTimer(time.Until(g.rate.Reset.Time)) | |||
select { | |||
case <-g.ctx.Done(): | |||
timer.Stop() | |||
return | |||
case <-timer.C: | |||
} | |||
rates, _, err := g.client.RateLimits(g.ctx) | |||
if err != nil { | |||
log.Error("g.client.RateLimits: %s", err) | |||
} | |||
g.rate = rates.GetCore() | |||
} | |||
} | |||
// GetRepoInfo returns a repository information | |||
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { | |||
gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) | |||
g.sleep() | |||
gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) | |||
if err != nil { | |||
return nil, err | |||
} | |||
g.rate = &resp.Rate | |||
// convert github repo to stand Repo | |||
return &base.Repository{ | |||
Owner: g.repoOwner, | |||
@@ -126,8 +155,13 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { | |||
// GetTopics return github topics | |||
func (g *GithubDownloaderV3) GetTopics() ([]string, error) { | |||
r, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) | |||
return r.Topics, err | |||
g.sleep() | |||
r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName) | |||
if err != nil { | |||
return nil, err | |||
} | |||
g.rate = &resp.Rate | |||
return r.Topics, nil | |||
} | |||
// GetMilestones returns milestones | |||
@@ -135,7 +169,8 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { | |||
var perPage = 100 | |||
var milestones = make([]*base.Milestone, 0, perPage) | |||
for i := 1; ; i++ { | |||
ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, | |||
g.sleep() | |||
ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, | |||
&github.MilestoneListOptions{ | |||
State: "all", | |||
ListOptions: github.ListOptions{ | |||
@@ -145,6 +180,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { | |||
if err != nil { | |||
return nil, err | |||
} | |||
g.rate = &resp.Rate | |||
for _, m := range ms { | |||
var desc string | |||
@@ -189,7 +225,8 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { | |||
var perPage = 100 | |||
var labels = make([]*base.Label, 0, perPage) | |||
for i := 1; ; i++ { | |||
ls, _, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, | |||
g.sleep() | |||
ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, | |||
&github.ListOptions{ | |||
Page: i, | |||
PerPage: perPage, | |||
@@ -197,6 +234,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { | |||
if err != nil { | |||
return nil, err | |||
} | |||
g.rate = &resp.Rate | |||
for _, label := range ls { | |||
labels = append(labels, convertGithubLabel(label)) | |||
@@ -260,7 +298,8 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { | |||
var perPage = 100 | |||
var releases = make([]*base.Release, 0, perPage) | |||
for i := 1; ; i++ { | |||
ls, _, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, | |||
g.sleep() | |||
ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, | |||
&github.ListOptions{ | |||
Page: i, | |||
PerPage: perPage, | |||
@@ -268,6 +307,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { | |||
if err != nil { | |||
return nil, err | |||
} | |||
g.rate = &resp.Rate | |||
for _, release := range ls { | |||
releases = append(releases, g.convertGithubRelease(release)) | |||
@@ -304,11 +344,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, | |||
} | |||
var allIssues = make([]*base.Issue, 0, perPage) | |||
issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) | |||
g.sleep() | |||
issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) | |||
if err != nil { | |||
return nil, false, fmt.Errorf("error while listing repos: %v", err) | |||
} | |||
g.rate = &resp.Rate | |||
for _, issue := range issues { | |||
if issue.IsPullRequest() { | |||
continue | |||
@@ -365,10 +406,12 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er | |||
}, | |||
} | |||
for { | |||
g.sleep() | |||
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt) | |||
if err != nil { | |||
return nil, fmt.Errorf("error while listing repos: %v", err) | |||
} | |||
g.rate = &resp.Rate | |||
for _, comment := range comments { | |||
var email string | |||
if comment.User.Email != nil { | |||
@@ -408,11 +451,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq | |||
}, | |||
} | |||
var allPRs = make([]*base.PullRequest, 0, perPage) | |||
prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) | |||
g.sleep() | |||
prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) | |||
if err != nil { | |||
return nil, fmt.Errorf("error while listing repos: %v", err) | |||
} | |||
g.rate = &resp.Rate | |||
for _, pr := range prs { | |||
var body string | |||
if pr.Body != nil { |
@@ -6,6 +6,7 @@ | |||
package migrations | |||
import ( | |||
"context" | |||
"fmt" | |||
"code.gitea.io/gitea/models" | |||
@@ -28,10 +29,10 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { | |||
} | |||
// MigrateRepository migrate repository according MigrateOptions | |||
func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { | |||
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { | |||
var ( | |||
downloader base.Downloader | |||
uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName) | |||
uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) | |||
theFactory base.DownloaderFactory | |||
) | |||
@@ -69,6 +70,8 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt | |||
downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) | |||
} | |||
downloader.SetContext(ctx) | |||
if err := migrateRepository(downloader, uploader, opts); err != nil { | |||
if err1 := uploader.Rollback(); err1 != nil { | |||
log.Error("rollback failed: %v", err1) |
@@ -11,6 +11,7 @@ import ( | |||
"strings" | |||
"code.gitea.io/gitea/models" | |||
"code.gitea.io/gitea/modules/graceful" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/migrations" | |||
"code.gitea.io/gitea/modules/notification" | |||
@@ -95,7 +96,7 @@ func runMigrateTask(t *models.Task) (err error) { | |||
} | |||
opts.MigrateToRepoID = t.RepoID | |||
repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts) | |||
repo, err := migrations.MigrateRepository(graceful.GetManager().HammerContext(), t.Doer, t.Owner.Name, *opts) | |||
if err == nil { | |||
log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) | |||
return nil |
@@ -18,6 +18,7 @@ import ( | |||
"code.gitea.io/gitea/modules/context" | |||
"code.gitea.io/gitea/modules/convert" | |||
"code.gitea.io/gitea/modules/git" | |||
"code.gitea.io/gitea/modules/graceful" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/migrations" | |||
"code.gitea.io/gitea/modules/notification" | |||
@@ -481,7 +482,7 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { | |||
} | |||
}() | |||
if _, err = migrations.MigrateRepository(ctx.User, ctxUser.Name, opts); err != nil { | |||
if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, ctxUser.Name, opts); err != nil { | |||
handleMigrateError(ctx, ctxUser, remoteAddr, err) | |||
return | |||
} |