aboutsummaryrefslogtreecommitdiffstats
path: root/services/migrations/github.go
diff options
context:
space:
mode:
Diffstat (limited to 'services/migrations/github.go')
-rw-r--r--services/migrations/github.go186
1 files changed, 102 insertions, 84 deletions
diff --git a/services/migrations/github.go b/services/migrations/github.go
index 604ab84b39..2ce11615c6 100644
--- a/services/migrations/github.go
+++ b/services/migrations/github.go
@@ -20,7 +20,7 @@ import (
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/structs"
- "github.com/google/go-github/v61/github"
+ "github.com/google/go-github/v71/github"
"golang.org/x/oauth2"
)
@@ -64,7 +64,6 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
// from github via APIv3
type GithubDownloaderV3 struct {
base.NullDownloader
- ctx context.Context
clients []*github.Client
baseURL string
repoOwner string
@@ -79,20 +78,19 @@ type GithubDownloaderV3 struct {
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
-func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
+func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
downloader := GithubDownloaderV3{
userName: userName,
baseURL: baseURL,
password: password,
- ctx: ctx,
repoOwner: repoOwner,
repoName: repoName,
maxPerPage: 100,
}
if token != "" {
- tokens := strings.Split(token, ",")
- for _, token := range tokens {
+ tokens := strings.SplitSeq(token, ",")
+ for token := range tokens {
token = strings.TrimSpace(token)
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
@@ -135,18 +133,13 @@ func (g *GithubDownloaderV3) LogString() string {
func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {
githubClient := github.NewClient(client)
if baseURL != "https://github.com" {
- githubClient, _ = github.NewClient(client).WithEnterpriseURLs(baseURL, baseURL)
+ githubClient, _ = githubClient.WithEnterpriseURLs(baseURL, baseURL)
}
g.clients = append(g.clients, githubClient)
g.rates = append(g.rates, nil)
}
-// SetContext set context
-func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
- g.ctx = ctx
-}
-
-func (g *GithubDownloaderV3) waitAndPickClient() {
+func (g *GithubDownloaderV3) waitAndPickClient(ctx context.Context) {
var recentIdx int
var maxRemaining int
for i := 0; i < len(g.clients); i++ {
@@ -160,13 +153,13 @@ func (g *GithubDownloaderV3) waitAndPickClient() {
for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {
timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))
select {
- case <-g.ctx.Done():
+ case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
- err := g.RefreshRate()
+ err := g.RefreshRate(ctx)
if err != nil {
log.Error("g.getClient().RateLimit.Get: %s", err)
}
@@ -174,8 +167,8 @@ func (g *GithubDownloaderV3) waitAndPickClient() {
}
// RefreshRate update the current rate (doesn't count in rate limit)
-func (g *GithubDownloaderV3) RefreshRate() error {
- rates, _, err := g.getClient().RateLimit.Get(g.ctx)
+func (g *GithubDownloaderV3) RefreshRate(ctx context.Context) error {
+ rates, _, err := g.getClient().RateLimit.Get(ctx)
if err != nil {
// if rate limit is not enabled, ignore it
if strings.Contains(err.Error(), "404") {
@@ -198,9 +191,9 @@ func (g *GithubDownloaderV3) setRate(rate *github.Rate) {
}
// GetRepoInfo returns a repository information
-func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
- g.waitAndPickClient()
- gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+func (g *GithubDownloaderV3) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
+ g.waitAndPickClient(ctx)
+ gr, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
@@ -219,9 +212,9 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
}
// GetTopics return github topics
-func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
- g.waitAndPickClient()
- r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+func (g *GithubDownloaderV3) GetTopics(ctx context.Context) ([]string, error) {
+ g.waitAndPickClient(ctx)
+ r, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
@@ -230,12 +223,12 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
}
// GetMilestones returns milestones
-func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
+func (g *GithubDownloaderV3) GetMilestones(ctx context.Context) ([]*base.Milestone, error) {
perPage := g.maxPerPage
milestones := make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
- g.waitAndPickClient()
- ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
+ g.waitAndPickClient(ctx)
+ ms, resp, err := g.getClient().Issues.ListMilestones(ctx, g.repoOwner, g.repoName,
&github.MilestoneListOptions{
State: "all",
ListOptions: github.ListOptions{
@@ -279,12 +272,12 @@ func convertGithubLabel(label *github.Label) *base.Label {
}
// GetLabels returns labels
-func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
+func (g *GithubDownloaderV3) GetLabels(ctx context.Context) ([]*base.Label, error) {
perPage := g.maxPerPage
labels := make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
- g.waitAndPickClient()
- ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
+ g.waitAndPickClient(ctx)
+ ls, resp, err := g.getClient().Issues.ListLabels(ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
@@ -304,7 +297,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
return labels, nil
}
-func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release {
+func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *github.RepositoryRelease) *base.Release {
// GitHub allows commitish to be a reference.
// In this case, we need to remove the prefix, i.e. convert "refs/heads/main" to "main".
targetCommitish := strings.TrimPrefix(rel.GetTargetCommitish(), git.BranchPrefix)
@@ -339,12 +332,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
Created: asset.CreatedAt.Time,
Updated: asset.UpdatedAt.Time,
DownloadFunc: func() (io.ReadCloser, error) {
- g.waitAndPickClient()
- readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
+ g.waitAndPickClient(ctx)
+ readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(ctx, g.repoOwner, g.repoName, assetID, nil)
if err != nil {
return nil, err
}
- if err := g.RefreshRate(); err != nil {
+ if err := g.RefreshRate(ctx); err != nil {
log.Error("g.getClient().RateLimits: %s", err)
}
@@ -364,13 +357,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
return io.NopCloser(strings.NewReader(redirectURL)), nil
}
- g.waitAndPickClient()
- req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
+ g.waitAndPickClient(ctx)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, redirectURL, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
- err1 := g.RefreshRate()
+ err1 := g.RefreshRate(ctx)
if err1 != nil {
log.Error("g.RefreshRate(): %s", err1)
}
@@ -385,12 +378,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
}
// GetReleases returns releases
-func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
+func (g *GithubDownloaderV3) GetReleases(ctx context.Context) ([]*base.Release, error) {
perPage := g.maxPerPage
releases := make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
- g.waitAndPickClient()
- ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
+ g.waitAndPickClient(ctx)
+ ls, resp, err := g.getClient().Repositories.ListReleases(ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
@@ -401,7 +394,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
g.setRate(&resp.Rate)
for _, release := range ls {
- releases = append(releases, g.convertGithubRelease(release))
+ releases = append(releases, g.convertGithubRelease(ctx, release))
}
if len(ls) < perPage {
break
@@ -411,7 +404,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
}
// GetIssues returns issues according start and limit
-func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
+func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
@@ -426,8 +419,8 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
}
allIssues := make([]*base.Issue, 0, perPage)
- g.waitAndPickClient()
- issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
+ g.waitAndPickClient(ctx)
+ issues, resp, err := g.getClient().Issues.ListByRepo(ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %w", err)
}
@@ -447,10 +440,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
- g.waitAndPickClient()
- res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
- Page: i,
- PerPage: perPage,
+ g.waitAndPickClient(ctx)
+ res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListReactionOptions{
+ ListOptions: github.ListOptions{
+ Page: i,
+ PerPage: perPage,
+ },
})
if err != nil {
return nil, false, err
@@ -503,12 +498,12 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool {
}
// GetComments returns comments according issueNumber
-func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) {
- comments, err := g.getComments(commentable)
+func (g *GithubDownloaderV3) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
+ comments, err := g.getComments(ctx, commentable)
return comments, false, err
}
-func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) {
+func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, error) {
var (
allComments = make([]*base.Comment, 0, g.maxPerPage)
created = "created"
@@ -522,8 +517,8 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.
},
}
for {
- g.waitAndPickClient()
- comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt)
+ g.waitAndPickClient(ctx)
+ comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err)
}
@@ -533,10 +528,12 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
- g.waitAndPickClient()
- res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
- Page: i,
- PerPage: g.maxPerPage,
+ g.waitAndPickClient(ctx)
+ res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListReactionOptions{
+ ListOptions: github.ListOptions{
+ Page: i,
+ PerPage: g.maxPerPage,
+ },
})
if err != nil {
return nil, err
@@ -576,7 +573,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.
}
// GetAllComments returns repository comments according page and perPageSize
-func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) {
+func (g *GithubDownloaderV3) GetAllComments(ctx context.Context, page, perPage int) ([]*base.Comment, bool, error) {
var (
allComments = make([]*base.Comment, 0, perPage)
created = "created"
@@ -594,8 +591,8 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
},
}
- g.waitAndPickClient()
- comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt)
+ g.waitAndPickClient(ctx)
+ comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, 0, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %w", err)
}
@@ -608,10 +605,12 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
- g.waitAndPickClient()
- res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
- Page: i,
- PerPage: g.maxPerPage,
+ g.waitAndPickClient(ctx)
+ res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListReactionOptions{
+ ListOptions: github.ListOptions{
+ Page: i,
+ PerPage: g.maxPerPage,
+ },
})
if err != nil {
return nil, false, err
@@ -648,7 +647,7 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
}
// GetPullRequests returns pull requests according page and perPage
-func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
+func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
@@ -662,8 +661,8 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
},
}
allPRs := make([]*base.PullRequest, 0, perPage)
- g.waitAndPickClient()
- prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
+ g.waitAndPickClient(ctx)
+ prs, resp, err := g.getClient().PullRequests.List(ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %w", err)
}
@@ -679,10 +678,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
- g.waitAndPickClient()
- res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
- Page: i,
- PerPage: perPage,
+ g.waitAndPickClient(ctx)
+ res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListReactionOptions{
+ ListOptions: github.ListOptions{
+ Page: i,
+ PerPage: perPage,
+ },
})
if err != nil {
return nil, false, err
@@ -702,7 +703,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
}
// download patch and saved as tmp file
- g.waitAndPickClient()
+ g.waitAndPickClient(ctx)
allPRs = append(allPRs, &base.PullRequest{
Title: pr.GetTitle(),
@@ -759,17 +760,19 @@ func convertGithubReview(r *github.PullRequestReview) *base.Review {
}
}
-func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
+func (g *GithubDownloaderV3) convertGithubReviewComments(ctx context.Context, cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
rcs := make([]*base.ReviewComment, 0, len(cs))
for _, c := range cs {
// get reactions
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
- g.waitAndPickClient()
- res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
- Page: i,
- PerPage: g.maxPerPage,
+ g.waitAndPickClient(ctx)
+ res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListReactionOptions{
+ ListOptions: github.ListOptions{
+ Page: i,
+ PerPage: g.maxPerPage,
+ },
})
if err != nil {
return nil, err
@@ -806,7 +809,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
}
// GetReviews returns pull requests review
-func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) {
+func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
allReviews := make([]*base.Review, 0, g.maxPerPage)
if g.SkipReviews {
return allReviews, nil
@@ -816,8 +819,8 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev
}
// Get approve/request change reviews
for {
- g.waitAndPickClient()
- reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
+ g.waitAndPickClient(ctx)
+ reviews, resp, err := g.getClient().PullRequests.ListReviews(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err)
}
@@ -830,14 +833,14 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev
PerPage: g.maxPerPage,
}
for {
- g.waitAndPickClient()
- reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2)
+ g.waitAndPickClient(ctx)
+ reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err)
}
g.setRate(&resp.Rate)
- cs, err := g.convertGithubReviewComments(reviewComments)
+ cs, err := g.convertGithubReviewComments(ctx, reviewComments)
if err != nil {
return nil, err
}
@@ -856,8 +859,8 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev
}
// Get requested reviews
for {
- g.waitAndPickClient()
- reviewers, resp, err := g.getClient().PullRequests.ListReviewers(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
+ g.waitAndPickClient(ctx)
+ reviewers, resp, err := g.getClient().PullRequests.ListReviewers(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err)
}
@@ -879,3 +882,18 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev
}
return allReviews, nil
}
+
+// FormatCloneURL add authentication into remote URLs
+func (g *GithubDownloaderV3) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
+ u, err := url.Parse(remoteAddr)
+ if err != nil {
+ return "", err
+ }
+ if len(opts.AuthToken) > 0 {
+ // "multiple tokens" are used to benefit more "API rate limit quota"
+ // git clone doesn't count for rate limits, so only use the first token.
+ // source: https://github.com/orgs/community/discussions/44515
+ u.User = url.UserPassword("oauth2", strings.Split(opts.AuthToken, ",")[0])
+ }
+ return u.String(), nil
+}
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 20:53:41 +0000