diff options
Diffstat (limited to 'services/migrations')
| -rw-r--r-- | services/migrations/codebase.go | 50 | ||||
| -rw-r--r-- | services/migrations/codebase_test.go | 19 | ||||
| -rw-r--r-- | services/migrations/codecommit.go | 48 | ||||
| -rw-r--r-- | services/migrations/dump.go | 48 | ||||
| -rw-r--r-- | services/migrations/error.go | 2 | ||||
| -rw-r--r-- | services/migrations/git.go | 8 | ||||
| -rw-r--r-- | services/migrations/gitea_downloader.go | 37 | ||||
| -rw-r--r-- | services/migrations/gitea_downloader_test.go | 31 | ||||
| -rw-r--r-- | services/migrations/gitea_uploader.go | 120 | ||||
| -rw-r--r-- | services/migrations/gitea_uploader_test.go | 55 | ||||
| -rw-r--r-- | services/migrations/github.go | 186 | ||||
| -rw-r--r-- | services/migrations/github_test.go | 60 | ||||
| -rw-r--r-- | services/migrations/gitlab.go | 72 | ||||
| -rw-r--r-- | services/migrations/gitlab_test.go | 36 | ||||
| -rw-r--r-- | services/migrations/gogs.go | 83 | ||||
| -rw-r--r-- | services/migrations/gogs_test.go | 17 | ||||
| -rw-r--r-- | services/migrations/migrate.go | 69 | ||||
| -rw-r--r-- | services/migrations/onedev.go | 52 | ||||
| -rw-r--r-- | services/migrations/onedev_test.go | 18 | ||||
| -rw-r--r-- | services/migrations/restore.go | 27 |
20 files changed, 527 insertions, 511 deletions
diff --git a/services/migrations/codebase.go b/services/migrations/codebase.go index 492fc908e9..240c7bcdc9 100644 --- a/services/migrations/codebase.go +++ b/services/migrations/codebase.go @@ -66,7 +66,6 @@ type codebaseUser struct { // from Codebase type CodebaseDownloader struct { base.NullDownloader - ctx context.Context client *http.Client baseURL *url.URL projectURL *url.URL @@ -77,17 +76,11 @@ type CodebaseDownloader struct { commitMap map[string]string } -// SetContext set context -func (d *CodebaseDownloader) SetContext(ctx context.Context) { - d.ctx = ctx -} - // NewCodebaseDownloader creates a new downloader -func NewCodebaseDownloader(ctx context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader { +func NewCodebaseDownloader(_ context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader { baseURL, _ := url.Parse("https://api3.codebasehq.com") downloader := &CodebaseDownloader{ - ctx: ctx, baseURL: baseURL, projectURL: projectURL, project: project, @@ -127,7 +120,7 @@ func (d *CodebaseDownloader) FormatCloneURL(opts base.MigrateOptions, remoteAddr return opts.CloneAddr, nil } -func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]string, result any) error { +func (d *CodebaseDownloader) callAPI(ctx context.Context, endpoint string, parameter map[string]string, result any) error { u, err := d.baseURL.Parse(endpoint) if err != nil { return err @@ -141,7 +134,7 @@ func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]strin u.RawQuery = query.Encode() } - req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return err } @@ -158,7 +151,7 @@ func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]strin // GetRepoInfo returns repository information // https://support.codebasehq.com/kb/projects -func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) { +func (d *CodebaseDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { var rawRepository struct { XMLName xml.Name `xml:"repository"` Name string `xml:"name"` @@ -169,6 +162,7 @@ func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) { } err := d.callAPI( + ctx, fmt.Sprintf("/%s/%s", d.project, d.repoName), nil, &rawRepository, @@ -187,7 +181,7 @@ func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) { // GetMilestones returns milestones // https://support.codebasehq.com/kb/tickets-and-milestones/milestones -func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) { +func (d *CodebaseDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { var rawMilestones struct { XMLName xml.Name `xml:"ticketing-milestone"` Type string `xml:"type,attr"` @@ -209,6 +203,7 @@ func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) { } err := d.callAPI( + ctx, fmt.Sprintf("/%s/milestones", d.project), nil, &rawMilestones, @@ -245,7 +240,7 @@ func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) { // GetLabels returns labels // https://support.codebasehq.com/kb/tickets-and-milestones/statuses-priorities-and-categories -func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) { +func (d *CodebaseDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) { var rawTypes struct { XMLName xml.Name `xml:"ticketing-types"` Type string `xml:"type,attr"` @@ -259,6 +254,7 @@ func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) { } err := d.callAPI( + ctx, fmt.Sprintf("/%s/tickets/types", d.project), nil, &rawTypes, @@ -284,7 +280,7 @@ type codebaseIssueContext struct { // GetIssues returns issues, limits are not supported // https://support.codebasehq.com/kb/tickets-and-milestones // https://support.codebasehq.com/kb/tickets-and-milestones/updating-tickets -func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (d *CodebaseDownloader) GetIssues(ctx context.Context, _, _ int) ([]*base.Issue, bool, error) { var rawIssues struct { XMLName xml.Name `xml:"tickets"` Type string `xml:"type,attr"` @@ -324,6 +320,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, } err := d.callAPI( + ctx, fmt.Sprintf("/%s/tickets", d.project), nil, &rawIssues, @@ -358,6 +355,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, } `xml:"ticket-note"` } err := d.callAPI( + ctx, fmt.Sprintf("/%s/tickets/%d/notes", d.project, issue.TicketID.Value), nil, ¬es, @@ -370,7 +368,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, if len(note.Content) == 0 { continue } - poster := d.tryGetUser(note.UserID.Value) + poster := d.tryGetUser(ctx, note.UserID.Value) comments = append(comments, &base.Comment{ IssueIndex: issue.TicketID.Value, Index: note.ID.Value, @@ -390,7 +388,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, if issue.Status.TreatAsClosed.Value { state = "closed" } - poster := d.tryGetUser(issue.ReporterID.Value) + poster := d.tryGetUser(ctx, issue.ReporterID.Value) issues = append(issues, &base.Issue{ Title: issue.Summary, Number: issue.TicketID.Value, @@ -419,7 +417,7 @@ func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, } // GetComments returns comments -func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (d *CodebaseDownloader) GetComments(_ context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { context, ok := commentable.GetContext().(codebaseIssueContext) if !ok { return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) @@ -430,7 +428,7 @@ func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base. // GetPullRequests returns pull requests // https://support.codebasehq.com/kb/repositories/merge-requests -func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (d *CodebaseDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { var rawMergeRequests struct { XMLName xml.Name `xml:"merge-requests"` Type string `xml:"type,attr"` @@ -443,6 +441,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq } err := d.callAPI( + ctx, fmt.Sprintf("/%s/%s/merge_requests", d.project, d.repoName), map[string]string{ "query": `"Target Project" is "` + d.repoName + `"`, @@ -503,6 +502,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq } `xml:"comments"` } err := d.callAPI( + ctx, fmt.Sprintf("/%s/%s/merge_requests/%d", d.project, d.repoName, mr.ID.Value), nil, &rawMergeRequest, @@ -531,7 +531,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq } continue } - poster := d.tryGetUser(comment.UserID.Value) + poster := d.tryGetUser(ctx, comment.UserID.Value) comments = append(comments, &base.Comment{ IssueIndex: number, Index: comment.ID.Value, @@ -547,7 +547,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq comments = append(comments, &base.Comment{}) } - poster := d.tryGetUser(rawMergeRequest.UserID.Value) + poster := d.tryGetUser(ctx, rawMergeRequest.UserID.Value) pullRequests = append(pullRequests, &base.PullRequest{ Title: rawMergeRequest.Subject, @@ -563,12 +563,12 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq MergedTime: mergedTime, Head: base.PullRequestBranch{ Ref: rawMergeRequest.SourceRef, - SHA: d.getHeadCommit(rawMergeRequest.SourceRef), + SHA: d.getHeadCommit(ctx, rawMergeRequest.SourceRef), RepoName: d.repoName, }, Base: base.PullRequestBranch{ Ref: rawMergeRequest.TargetRef, - SHA: d.getHeadCommit(rawMergeRequest.TargetRef), + SHA: d.getHeadCommit(ctx, rawMergeRequest.TargetRef), RepoName: d.repoName, }, ForeignIndex: rawMergeRequest.ID.Value, @@ -584,7 +584,7 @@ func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullReq return pullRequests, true, nil } -func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { +func (d *CodebaseDownloader) tryGetUser(ctx context.Context, userID int64) *codebaseUser { if len(d.userMap) == 0 { var rawUsers struct { XMLName xml.Name `xml:"users"` @@ -602,6 +602,7 @@ func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { } err := d.callAPI( + ctx, "/users", nil, &rawUsers, @@ -627,7 +628,7 @@ func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { return user } -func (d *CodebaseDownloader) getHeadCommit(ref string) string { +func (d *CodebaseDownloader) getHeadCommit(ctx context.Context, ref string) string { commitRef, ok := d.commitMap[ref] if !ok { var rawCommits struct { @@ -638,6 +639,7 @@ func (d *CodebaseDownloader) getHeadCommit(ref string) string { } `xml:"commit"` } err := d.callAPI( + ctx, fmt.Sprintf("/%s/%s/commits/%s", d.project, d.repoName, ref), nil, &rawCommits, diff --git a/services/migrations/codebase_test.go b/services/migrations/codebase_test.go index 68721e0641..6cd52e5e59 100644 --- a/services/migrations/codebase_test.go +++ b/services/migrations/codebase_test.go @@ -4,7 +4,6 @@ package migrations import ( - "context" "net/url" "os" "testing" @@ -30,9 +29,9 @@ func TestCodebaseDownloadRepo(t *testing.T) { if cloneUser != "" { u.User = url.UserPassword(cloneUser, clonePassword) } - + ctx := t.Context() factory := &CodebaseDownloaderFactory{} - downloader, err := factory.New(context.Background(), base.MigrateOptions{ + downloader, err := factory.New(ctx, base.MigrateOptions{ CloneAddr: u.String(), AuthUsername: apiUser, AuthPassword: apiPassword, @@ -40,7 +39,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { if err != nil { t.Fatalf("Error creating Codebase downloader: %v", err) } - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) assertRepositoryEqual(t, &base.Repository{ Name: "test", @@ -50,7 +49,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { OriginalURL: cloneAddr, }, repo) - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) assertMilestonesEqual(t, []*base.Milestone{ { @@ -65,11 +64,11 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, milestones) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assert.Len(t, labels, 4) - issues, isEnd, err := downloader.GetIssues(1, 2) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 2) assert.NoError(t, err) assert.True(t, isEnd) assertIssuesEqual(t, []*base.Issue{ @@ -106,7 +105,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(issues[0]) + comments, _, err := downloader.GetComments(ctx, issues[0]) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -119,7 +118,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, comments) - prs, _, err := downloader.GetPullRequests(1, 1) + prs, _, err := downloader.GetPullRequests(ctx, 1, 1) assert.NoError(t, err) assertPullRequestsEqual(t, []*base.PullRequest{ { @@ -144,7 +143,7 @@ func TestCodebaseDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(prs[0]) + rvs, err := downloader.GetReviews(ctx, prs[0]) assert.NoError(t, err) assert.Empty(t, rvs) } diff --git a/services/migrations/codecommit.go b/services/migrations/codecommit.go index fead527f5b..d08b2e6d4a 100644 --- a/services/migrations/codecommit.go +++ b/services/migrations/codecommit.go @@ -5,7 +5,7 @@ package migrations import ( "context" - "fmt" + "errors" "net/url" "strconv" "strings" @@ -42,13 +42,13 @@ func (c *CodeCommitDownloaderFactory) New(ctx context.Context, opts base.Migrate hostElems := strings.Split(u.Host, ".") if len(hostElems) != 4 { - return nil, fmt.Errorf("cannot get the region from clone URL") + return nil, errors.New("cannot get the region from clone URL") } region := hostElems[1] pathElems := strings.Split(u.Path, "/") if len(pathElems) == 0 { - return nil, fmt.Errorf("cannot get the repo name from clone URL") + return nil, errors.New("cannot get the repo name from clone URL") } repoName := pathElems[len(pathElems)-1] @@ -62,9 +62,8 @@ func (c *CodeCommitDownloaderFactory) GitServiceType() structs.GitServiceType { return structs.CodeCommitService } -func NewCodeCommitDownloader(ctx context.Context, repoName, baseURL, accessKeyID, secretAccessKey, region string) *CodeCommitDownloader { +func NewCodeCommitDownloader(_ context.Context, repoName, baseURL, accessKeyID, secretAccessKey, region string) *CodeCommitDownloader { downloader := CodeCommitDownloader{ - ctx: ctx, repoName: repoName, baseURL: baseURL, codeCommitClient: codecommit.New(codecommit.Options{ @@ -79,21 +78,15 @@ func NewCodeCommitDownloader(ctx context.Context, repoName, baseURL, accessKeyID // CodeCommitDownloader implements a downloader for AWS CodeCommit type CodeCommitDownloader struct { base.NullDownloader - ctx context.Context codeCommitClient *codecommit.Client repoName string baseURL string allPullRequestIDs []string } -// SetContext set context -func (c *CodeCommitDownloader) SetContext(ctx context.Context) { - c.ctx = ctx -} - // GetRepoInfo returns a repository information -func (c *CodeCommitDownloader) GetRepoInfo() (*base.Repository, error) { - output, err := c.codeCommitClient.GetRepository(c.ctx, &codecommit.GetRepositoryInput{ +func (c *CodeCommitDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { + output, err := c.codeCommitClient.GetRepository(ctx, &codecommit.GetRepositoryInput{ RepositoryName: util.ToPointer(c.repoName), }) if err != nil { @@ -117,14 +110,14 @@ func (c *CodeCommitDownloader) GetRepoInfo() (*base.Repository, error) { } // GetComments returns comments of an issue or PR -func (c *CodeCommitDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (c *CodeCommitDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { var ( nextToken *string comments []*base.Comment ) for { - resp, err := c.codeCommitClient.GetCommentsForPullRequest(c.ctx, &codecommit.GetCommentsForPullRequestInput{ + resp, err := c.codeCommitClient.GetCommentsForPullRequest(ctx, &codecommit.GetCommentsForPullRequestInput{ NextToken: nextToken, PullRequestId: util.ToPointer(strconv.FormatInt(commentable.GetForeignIndex(), 10)), }) @@ -155,22 +148,19 @@ func (c *CodeCommitDownloader) GetComments(commentable base.Commentable) ([]*bas } // GetPullRequests returns pull requests according page and perPage -func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { - allPullRequestIDs, err := c.getAllPullRequestIDs() +func (c *CodeCommitDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { + allPullRequestIDs, err := c.getAllPullRequestIDs(ctx) if err != nil { return nil, false, err } startIndex := (page - 1) * perPage - endIndex := page * perPage - if endIndex > len(allPullRequestIDs) { - endIndex = len(allPullRequestIDs) - } + endIndex := min(page*perPage, len(allPullRequestIDs)) batch := allPullRequestIDs[startIndex:endIndex] prs := make([]*base.PullRequest, 0, len(batch)) for _, id := range batch { - output, err := c.codeCommitClient.GetPullRequest(c.ctx, &codecommit.GetPullRequestInput{ + output, err := c.codeCommitClient.GetPullRequest(ctx, &codecommit.GetPullRequestInput{ PullRequestId: util.ToPointer(id), }) if err != nil { @@ -187,11 +177,15 @@ func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullR continue } target := orig.PullRequestTargets[0] + description := "" + if orig.Description != nil { + description = *orig.Description + } pr := &base.PullRequest{ Number: number, Title: *orig.Title, PosterName: c.getUsernameFromARN(*orig.AuthorArn), - Content: *orig.Description, + Content: description, State: "open", Created: *orig.CreationDate, Updated: *orig.LastActivityDate, @@ -213,6 +207,10 @@ func (c *CodeCommitDownloader) GetPullRequests(page, perPage int) ([]*base.PullR pr.State = "closed" pr.Closed = orig.LastActivityDate } + if pr.Merged { + pr.MergeCommitSHA = *target.MergeMetadata.MergeCommitId + pr.MergedTime = orig.LastActivityDate + } _ = CheckAndEnsureSafePR(pr, c.baseURL, c) prs = append(prs, pr) @@ -231,7 +229,7 @@ func (c *CodeCommitDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr st return u.String(), nil } -func (c *CodeCommitDownloader) getAllPullRequestIDs() ([]string, error) { +func (c *CodeCommitDownloader) getAllPullRequestIDs(ctx context.Context) ([]string, error) { if len(c.allPullRequestIDs) > 0 { return c.allPullRequestIDs, nil } @@ -242,7 +240,7 @@ func (c *CodeCommitDownloader) getAllPullRequestIDs() ([]string, error) { ) for { - output, err := c.codeCommitClient.ListPullRequests(c.ctx, &codecommit.ListPullRequestsInput{ + output, err := c.codeCommitClient.ListPullRequests(ctx, &codecommit.ListPullRequestsInput{ RepositoryName: util.ToPointer(c.repoName), NextToken: nextToken, }) diff --git a/services/migrations/dump.go b/services/migrations/dump.go index 07812002af..b4ca1e41e0 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -32,7 +32,6 @@ var _ base.Uploader = &RepositoryDumper{} // RepositoryDumper implements an Uploader to the local directory type RepositoryDumper struct { - ctx context.Context baseDir string repoOwner string repoName string @@ -56,7 +55,6 @@ func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName strin return nil, err } return &RepositoryDumper{ - ctx: ctx, opts: opts, baseDir: baseDir, repoOwner: repoOwner, @@ -105,7 +103,7 @@ func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) { } // CreateRepo creates a repository -func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { +func (g *RepositoryDumper) CreateRepo(ctx context.Context, repo *base.Repository, opts base.MigrateOptions) error { f, err := os.Create(filepath.Join(g.baseDir, "repo.yml")) if err != nil { return err @@ -149,7 +147,7 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp return err } - err = git.Clone(g.ctx, remoteAddr, repoPath, git.CloneRepoOptions{ + err = git.Clone(ctx, remoteAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -158,19 +156,19 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp if err != nil { return fmt.Errorf("Clone: %w", err) } - if err := git.WriteCommitGraph(g.ctx, repoPath); err != nil { + if err := git.WriteCommitGraph(ctx, repoPath); err != nil { return err } if opts.Wiki { wikiPath := g.wikiPath() - wikiRemotePath := repository.WikiRemoteURL(g.ctx, remoteAddr) + wikiRemotePath := repository.WikiRemoteURL(ctx, remoteAddr) if len(wikiRemotePath) > 0 { if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil { return fmt.Errorf("Failed to remove %s: %w", wikiPath, err) } - if err := git.Clone(g.ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ + if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -181,13 +179,13 @@ func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOp if err := os.RemoveAll(wikiPath); err != nil { return fmt.Errorf("Failed to remove %s: %w", wikiPath, err) } - } else if err := git.WriteCommitGraph(g.ctx, wikiPath); err != nil { + } else if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { return err } } } - g.gitRepo, err = git.OpenRepository(g.ctx, g.gitPath()) + g.gitRepo, err = git.OpenRepository(ctx, g.gitPath()) return err } @@ -220,7 +218,7 @@ func (g *RepositoryDumper) Close() { } // CreateTopics creates topics -func (g *RepositoryDumper) CreateTopics(topics ...string) error { +func (g *RepositoryDumper) CreateTopics(_ context.Context, topics ...string) error { f, err := os.Create(filepath.Join(g.baseDir, "topic.yml")) if err != nil { return err @@ -242,7 +240,7 @@ func (g *RepositoryDumper) CreateTopics(topics ...string) error { } // CreateMilestones creates milestones -func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error { +func (g *RepositoryDumper) CreateMilestones(_ context.Context, milestones ...*base.Milestone) error { var err error if g.milestoneFile == nil { g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml")) @@ -264,7 +262,7 @@ func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error } // CreateLabels creates labels -func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { +func (g *RepositoryDumper) CreateLabels(_ context.Context, labels ...*base.Label) error { var err error if g.labelFile == nil { g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml")) @@ -286,7 +284,7 @@ func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error { } // CreateReleases creates releases -func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { +func (g *RepositoryDumper) CreateReleases(_ context.Context, releases ...*base.Release) error { if g.opts.ReleaseAssets { for _, release := range releases { attachDir := filepath.Join("release_assets", release.TagName) @@ -354,12 +352,12 @@ func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error { } // SyncTags syncs releases with tags in the database -func (g *RepositoryDumper) SyncTags() error { +func (g *RepositoryDumper) SyncTags(ctx context.Context) error { return nil } // CreateIssues creates issues -func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error { +func (g *RepositoryDumper) CreateIssues(_ context.Context, issues ...*base.Issue) error { var err error if g.issueFile == nil { g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml")) @@ -412,7 +410,7 @@ func (g *RepositoryDumper) encodeItems(number int64, items []any, dir string, it } // CreateComments creates comments of issues -func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { +func (g *RepositoryDumper) CreateComments(_ context.Context, comments ...*base.Comment) error { commentsMap := make(map[int64][]any, len(comments)) for _, comment := range comments { commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment) @@ -421,7 +419,7 @@ func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error { return g.createItems(g.commentDir(), g.commentFiles, commentsMap) } -func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { +func (g *RepositoryDumper) handlePullRequest(ctx context.Context, pr *base.PullRequest) error { // SECURITY: this pr must have been ensured safe if !pr.EnsuredSafe { log.Error("PR #%d in %s/%s has not been checked for safety ... We will ignore this.", pr.Number, g.repoOwner, g.repoName) @@ -490,7 +488,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { if pr.Head.CloneURL == "" || pr.Head.Ref == "" { // Set head information if pr.Head.SHA is available if pr.Head.SHA != "" { - _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + _, _, err = git.NewCommand("update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.gitPath()}) if err != nil { log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err) } @@ -520,7 +518,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { if !ok { // Set head information if pr.Head.SHA is available if pr.Head.SHA != "" { - _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + _, _, err = git.NewCommand("update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.gitPath()}) if err != nil { log.Error("PR #%d in %s/%s unable to update-ref for pr HEAD: %v", pr.Number, g.repoOwner, g.repoName, err) } @@ -555,7 +553,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { fetchArg = git.BranchPrefix + fetchArg } - _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + _, _, err = git.NewCommand("fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(ctx, &git.RunOpts{Dir: g.gitPath()}) if err != nil { log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) // We need to continue here so that the Head.Ref is reset and we attempt to set the gitref for the PR @@ -579,7 +577,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { pr.Head.SHA = headSha } if pr.Head.SHA != "" { - _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.gitPath()}) + _, _, err = git.NewCommand("update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.gitPath()}) if err != nil { log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err) } @@ -589,7 +587,7 @@ func (g *RepositoryDumper) handlePullRequest(pr *base.PullRequest) error { } // CreatePullRequests creates pull requests -func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { +func (g *RepositoryDumper) CreatePullRequests(ctx context.Context, prs ...*base.PullRequest) error { var err error if g.pullrequestFile == nil { if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil { @@ -607,7 +605,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { count := 0 for i := 0; i < len(prs); i++ { pr := prs[i] - if err := g.handlePullRequest(pr); err != nil { + if err := g.handlePullRequest(ctx, pr); err != nil { log.Error("PR #%d in %s/%s failed - skipping", pr.Number, g.repoOwner, g.repoName, err) continue } @@ -620,7 +618,7 @@ func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error { } // CreateReviews create pull request reviews -func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error { +func (g *RepositoryDumper) CreateReviews(_ context.Context, reviews ...*base.Review) error { reviewsMap := make(map[int64][]any, len(reviews)) for _, review := range reviews { reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review) @@ -636,7 +634,7 @@ func (g *RepositoryDumper) Rollback() error { } // Finish when migrating succeed, this will update something. -func (g *RepositoryDumper) Finish() error { +func (g *RepositoryDumper) Finish(_ context.Context) error { return nil } diff --git a/services/migrations/error.go b/services/migrations/error.go index c7d912f50b..9b470149bf 100644 --- a/services/migrations/error.go +++ b/services/migrations/error.go @@ -7,7 +7,7 @@ package migrations import ( "errors" - "github.com/google/go-github/v61/github" + "github.com/google/go-github/v71/github" ) // ErrRepoNotCreated returns the error that repository not created diff --git a/services/migrations/git.go b/services/migrations/git.go index 22ffd5e765..1ed99499a1 100644 --- a/services/migrations/git.go +++ b/services/migrations/git.go @@ -28,12 +28,8 @@ func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownl } } -// SetContext set context -func (g *PlainGitDownloader) SetContext(ctx context.Context) { -} - // GetRepoInfo returns a repository information -func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { +func (g *PlainGitDownloader) GetRepoInfo(_ context.Context) (*base.Repository, error) { // convert github repo to stand Repo return &base.Repository{ Owner: g.ownerName, @@ -43,6 +39,6 @@ func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) { } // GetTopics return empty string slice -func (g PlainGitDownloader) GetTopics() ([]string, error) { +func (g PlainGitDownloader) GetTopics(_ context.Context) ([]string, error) { return []string{}, nil } diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index 272bf02e11..5d48d2f003 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -67,7 +67,6 @@ func (f *GiteaDownloaderFactory) GitServiceType() structs.GitServiceType { // GiteaDownloader implements a Downloader interface to get repository information's type GiteaDownloader struct { base.NullDownloader - ctx context.Context client *gitea_sdk.Client baseURL string repoOwner string @@ -114,7 +113,6 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo } return &GiteaDownloader{ - ctx: ctx, client: giteaClient, baseURL: baseURL, repoOwner: path[0], @@ -124,11 +122,6 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo }, nil } -// SetContext set context -func (g *GiteaDownloader) SetContext(ctx context.Context) { - g.ctx = ctx -} - // String implements Stringer func (g *GiteaDownloader) String() string { return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) @@ -142,7 +135,7 @@ func (g *GiteaDownloader) LogString() string { } // GetRepoInfo returns a repository information -func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) { +func (g *GiteaDownloader) GetRepoInfo(_ context.Context) (*base.Repository, error) { if g == nil { return nil, errors.New("error: GiteaDownloader is nil") } @@ -164,19 +157,19 @@ func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) { } // GetTopics return gitea topics -func (g *GiteaDownloader) GetTopics() ([]string, error) { +func (g *GiteaDownloader) GetTopics(_ context.Context) ([]string, error) { topics, _, err := g.client.ListRepoTopics(g.repoOwner, g.repoName, gitea_sdk.ListRepoTopicsOptions{}) return topics, err } // GetMilestones returns milestones -func (g *GiteaDownloader) GetMilestones() ([]*base.Milestone, error) { +func (g *GiteaDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { milestones := make([]*base.Milestone, 0, g.maxPerPage) for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { - case <-g.ctx.Done(): + case <-ctx.Done(): return nil, nil default: } @@ -235,13 +228,13 @@ func (g *GiteaDownloader) convertGiteaLabel(label *gitea_sdk.Label) *base.Label } // GetLabels returns labels -func (g *GiteaDownloader) GetLabels() ([]*base.Label, error) { +func (g *GiteaDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) { labels := make([]*base.Label, 0, g.maxPerPage) for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { - case <-g.ctx.Done(): + case <-ctx.Done(): return nil, nil default: } @@ -305,7 +298,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele } // FIXME: for a private download? - req, err := http.NewRequest("GET", assetDownloadURL, nil) + req, err := http.NewRequest(http.MethodGet, assetDownloadURL, nil) if err != nil { return nil, err } @@ -323,13 +316,13 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele } // GetReleases returns releases -func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) { +func (g *GiteaDownloader) GetReleases(ctx context.Context) ([]*base.Release, error) { releases := make([]*base.Release, 0, g.maxPerPage) for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { - case <-g.ctx.Done(): + case <-ctx.Done(): return nil, nil default: } @@ -395,7 +388,7 @@ func (g *GiteaDownloader) getCommentReactions(commentID int64) ([]*base.Reaction } // GetIssues returns issues according start and limit -func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (g *GiteaDownloader) GetIssues(_ context.Context, page, perPage int) ([]*base.Issue, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -458,13 +451,13 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err } // GetComments returns comments according issueNumber -func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (g *GiteaDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, g.maxPerPage) for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { - case <-g.ctx.Done(): + case <-ctx.Done(): return nil, false, nil default: } @@ -504,7 +497,7 @@ func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Com } // GetPullRequests returns pull requests according page and perPage -func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (g *GiteaDownloader) GetPullRequests(_ context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -624,7 +617,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques } // GetReviews returns pull requests review -func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (g *GiteaDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil { log.Info("GiteaDownloader: instance to old, skip GetReviews") return nil, nil @@ -635,7 +628,7 @@ func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { - case <-g.ctx.Done(): + case <-ctx.Done(): return nil, nil default: } diff --git a/services/migrations/gitea_downloader_test.go b/services/migrations/gitea_downloader_test.go index 6f6ef99d96..bb1760e889 100644 --- a/services/migrations/gitea_downloader_test.go +++ b/services/migrations/gitea_downloader_test.go @@ -4,7 +4,6 @@ package migrations import ( - "context" "net/http" "os" "sort" @@ -28,12 +27,12 @@ func TestGiteaDownloadRepo(t *testing.T) { if err != nil || resp.StatusCode != http.StatusOK { t.Skipf("Can't reach https://gitea.com, skipping %s", t.Name()) } - - downloader, err := NewGiteaDownloader(context.Background(), "https://gitea.com", "gitea/test_repo", "", "", giteaToken) + ctx := t.Context() + downloader, err := NewGiteaDownloader(ctx, "https://gitea.com", "gitea/test_repo", "", "", giteaToken) require.NoError(t, err, "NewGiteaDownloader error occur") require.NotNil(t, downloader, "NewGiteaDownloader is nil") - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) assertRepositoryEqual(t, &base.Repository{ Name: "test_repo", @@ -45,12 +44,12 @@ func TestGiteaDownloadRepo(t *testing.T) { DefaultBranch: "master", }, repo) - topics, err := downloader.GetTopics() + topics, err := downloader.GetTopics(ctx) assert.NoError(t, err) sort.Strings(topics) - assert.EqualValues(t, []string{"ci", "gitea", "migration", "test"}, topics) + assert.Equal(t, []string{"ci", "gitea", "migration", "test"}, topics) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assertLabelsEqual(t, []*base.Label{ { @@ -80,7 +79,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, labels) - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) assertMilestonesEqual(t, []*base.Milestone{ { @@ -100,7 +99,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, milestones) - releases, err := downloader.GetReleases() + releases, err := downloader.GetReleases(ctx) assert.NoError(t, err) assertReleasesEqual(t, []*base.Release{ { @@ -131,13 +130,13 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, releases) - issues, isEnd, err := downloader.GetIssues(1, 50) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 50) assert.NoError(t, err) assert.True(t, isEnd) assert.Len(t, issues, 7) - assert.EqualValues(t, "open", issues[0].State) + assert.Equal(t, "open", issues[0].State) - issues, isEnd, err = downloader.GetIssues(3, 2) + issues, isEnd, err = downloader.GetIssues(ctx, 3, 2) assert.NoError(t, err) assert.False(t, isEnd) @@ -194,7 +193,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{Number: 4, ForeignIndex: 4}) + comments, _, err := downloader.GetComments(ctx, &base.Issue{Number: 4, ForeignIndex: 4}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -217,11 +216,11 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, comments) - prs, isEnd, err := downloader.GetPullRequests(1, 50) + prs, isEnd, err := downloader.GetPullRequests(ctx, 1, 50) assert.NoError(t, err) assert.True(t, isEnd) assert.Len(t, prs, 6) - prs, isEnd, err = downloader.GetPullRequests(1, 3) + prs, isEnd, err = downloader.GetPullRequests(ctx, 1, 3) assert.NoError(t, err) assert.False(t, isEnd) assert.Len(t, prs, 3) @@ -259,7 +258,7 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, err := downloader.GetReviews(&base.Issue{Number: 7, ForeignIndex: 7}) + reviews, err := downloader.GetReviews(ctx, &base.Issue{Number: 7, ForeignIndex: 7}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 9e06b77b66..737bff24d0 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -41,7 +41,6 @@ var _ base.Uploader = &GiteaLocalUploader{} // GiteaLocalUploader implements an Uploader to gitea sites type GiteaLocalUploader struct { - ctx context.Context doer *user_model.User repoOwner string repoName string @@ -58,9 +57,8 @@ type GiteaLocalUploader struct { } // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1 -func NewGiteaLocalUploader(ctx context.Context, doer *user_model.User, repoOwner, repoName string) *GiteaLocalUploader { +func NewGiteaLocalUploader(_ context.Context, doer *user_model.User, repoOwner, repoName string) *GiteaLocalUploader { return &GiteaLocalUploader{ - ctx: ctx, doer: doer, repoOwner: repoOwner, repoName: repoName, @@ -93,15 +91,15 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int { } // CreateRepo creates a repository -func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error { - owner, err := user_model.GetUserByName(g.ctx, g.repoOwner) +func (g *GiteaLocalUploader) CreateRepo(ctx context.Context, repo *base.Repository, opts base.MigrateOptions) error { + owner, err := user_model.GetUserByName(ctx, g.repoOwner) if err != nil { return err } var r *repo_model.Repository if opts.MigrateToRepoID <= 0 { - r, err = repo_service.CreateRepositoryDirectly(g.ctx, g.doer, owner, repo_service.CreateRepoOptions{ + r, err = repo_service.CreateRepositoryDirectly(ctx, g.doer, owner, repo_service.CreateRepoOptions{ Name: g.repoName, Description: repo.Description, OriginalURL: repo.OriginalURL, @@ -109,9 +107,9 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate IsPrivate: opts.Private || setting.Repository.ForcePrivate, IsMirror: opts.Mirror, Status: repo_model.RepositoryBeingMigrated, - }) + }, false) } else { - r, err = repo_model.GetRepositoryByID(g.ctx, opts.MigrateToRepoID) + r, err = repo_model.GetRepositoryByID(ctx, opts.MigrateToRepoID) } if err != nil { return err @@ -119,7 +117,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate r.DefaultBranch = repo.DefaultBranch r.Description = repo.Description - r, err = repo_service.MigrateRepositoryGitData(g.ctx, owner, r, base.MigrateOptions{ + r, err = repo_service.MigrateRepositoryGitData(ctx, owner, r, base.MigrateOptions{ RepoName: g.repoName, Description: repo.Description, OriginalURL: repo.OriginalURL, @@ -139,7 +137,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate if err != nil { return err } - g.gitRepo, err = gitrepo.OpenRepository(g.ctx, g.repo) + g.gitRepo, err = gitrepo.OpenRepository(ctx, g.repo) if err != nil { return err } @@ -150,7 +148,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate return err } g.repo.ObjectFormatName = objectFormat.Name() - return repo_model.UpdateRepositoryCols(g.ctx, g.repo, "object_format_name") + return repo_model.UpdateRepositoryColsNoAutoTime(ctx, g.repo, "object_format_name") } // Close closes this uploader @@ -161,7 +159,7 @@ func (g *GiteaLocalUploader) Close() { } // CreateTopics creates topics -func (g *GiteaLocalUploader) CreateTopics(topics ...string) error { +func (g *GiteaLocalUploader) CreateTopics(ctx context.Context, topics ...string) error { // Ignore topics too long for the db c := 0 for _, topic := range topics { @@ -173,11 +171,11 @@ func (g *GiteaLocalUploader) CreateTopics(topics ...string) error { c++ } topics = topics[:c] - return repo_model.SaveTopics(g.ctx, g.repo.ID, topics...) + return repo_model.SaveTopics(ctx, g.repo.ID, topics...) } // CreateMilestones creates milestones -func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error { +func (g *GiteaLocalUploader) CreateMilestones(ctx context.Context, milestones ...*base.Milestone) error { mss := make([]*issues_model.Milestone, 0, len(milestones)) for _, milestone := range milestones { var deadline timeutil.TimeStamp @@ -216,7 +214,7 @@ func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) err mss = append(mss, &ms) } - err := issues_model.InsertMilestones(g.ctx, mss...) + err := issues_model.InsertMilestones(ctx, mss...) if err != nil { return err } @@ -228,7 +226,7 @@ func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) err } // CreateLabels creates labels -func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { +func (g *GiteaLocalUploader) CreateLabels(ctx context.Context, labels ...*base.Label) error { lbs := make([]*issues_model.Label, 0, len(labels)) for _, l := range labels { if color, err := label.NormalizeColor(l.Color); err != nil { @@ -247,7 +245,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { }) } - err := issues_model.NewLabels(g.ctx, lbs...) + err := issues_model.NewLabels(ctx, lbs...) if err != nil { return err } @@ -258,7 +256,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { } // CreateReleases creates releases -func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { +func (g *GiteaLocalUploader) CreateReleases(ctx context.Context, releases ...*base.Release) error { rels := make([]*repo_model.Release, 0, len(releases)) for _, release := range releases { if release.Created.IsZero() { @@ -292,7 +290,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { CreatedUnix: timeutil.TimeStamp(release.Created.Unix()), } - if err := g.remapUser(release, &rel); err != nil { + if err := g.remapUser(ctx, release, &rel); err != nil { return err } @@ -361,16 +359,16 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { rels = append(rels, &rel) } - return repo_model.InsertReleases(g.ctx, rels...) + return repo_model.InsertReleases(ctx, rels...) } // SyncTags syncs releases with tags in the database -func (g *GiteaLocalUploader) SyncTags() error { - return repo_module.SyncReleasesWithTags(g.ctx, g.repo, g.gitRepo) +func (g *GiteaLocalUploader) SyncTags(ctx context.Context) error { + return repo_module.SyncReleasesWithTags(ctx, g.repo, g.gitRepo) } // CreateIssues creates issues -func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { +func (g *GiteaLocalUploader) CreateIssues(ctx context.Context, issues ...*base.Issue) error { iss := make([]*issues_model.Issue, 0, len(issues)) for _, issue := range issues { var labels []*issues_model.Label @@ -419,7 +417,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()), } - if err := g.remapUser(issue, &is); err != nil { + if err := g.remapUser(ctx, issue, &is); err != nil { return err } @@ -432,7 +430,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { Type: reaction.Content, CreatedUnix: timeutil.TimeStampNow(), } - if err := g.remapUser(reaction, &res); err != nil { + if err := g.remapUser(ctx, reaction, &res); err != nil { return err } is.Reactions = append(is.Reactions, &res) @@ -441,7 +439,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } if len(iss) > 0 { - if err := issues_model.InsertIssues(g.ctx, iss...); err != nil { + if err := issues_model.InsertIssues(ctx, iss...); err != nil { return err } @@ -454,7 +452,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } // CreateComments creates comments of issues -func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { +func (g *GiteaLocalUploader) CreateComments(ctx context.Context, comments ...*base.Comment) error { cms := make([]*issues_model.Comment, 0, len(comments)) for _, comment := range comments { var issue *issues_model.Issue @@ -513,7 +511,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { default: } - if err := g.remapUser(comment, &cm); err != nil { + if err := g.remapUser(ctx, comment, &cm); err != nil { return err } @@ -523,7 +521,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { Type: reaction.Content, CreatedUnix: timeutil.TimeStampNow(), } - if err := g.remapUser(reaction, &res); err != nil { + if err := g.remapUser(ctx, reaction, &res); err != nil { return err } cm.Reactions = append(cm.Reactions, &res) @@ -535,35 +533,35 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { if len(cms) == 0 { return nil } - return issues_model.InsertIssueComments(g.ctx, cms) + return issues_model.InsertIssueComments(ctx, cms) } // CreatePullRequests creates pull requests -func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error { +func (g *GiteaLocalUploader) CreatePullRequests(ctx context.Context, prs ...*base.PullRequest) error { gprs := make([]*issues_model.PullRequest, 0, len(prs)) for _, pr := range prs { - gpr, err := g.newPullRequest(pr) + gpr, err := g.newPullRequest(ctx, pr) if err != nil { return err } - if err := g.remapUser(pr, gpr.Issue); err != nil { + if err := g.remapUser(ctx, pr, gpr.Issue); err != nil { return err } gprs = append(gprs, gpr) } - if err := issues_model.InsertPullRequests(g.ctx, gprs...); err != nil { + if err := issues_model.InsertPullRequests(ctx, gprs...); err != nil { return err } for _, pr := range gprs { g.issues[pr.Issue.Index] = pr.Issue - pull.AddToTaskQueue(g.ctx, pr) + pull.StartPullRequestCheckImmediately(ctx, pr) } return nil } -func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) { +func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *base.PullRequest) (head string, err error) { // SECURITY: this pr must have been must have been ensured safe if !pr.EnsuredSafe { log.Error("PR #%d in %s/%s has not been checked for safety.", pr.Number, g.repoOwner, g.repoName) @@ -664,7 +662,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head fetchArg = git.BranchPrefix + fetchArg } - _, _, err = git.NewCommand(g.ctx, "fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + _, _, err = git.NewCommand("fetch", "--no-tags").AddDashesAndList(remote, fetchArg).RunStdString(ctx, &git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err) return head, nil @@ -683,7 +681,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head pr.Head.SHA = headSha } - _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + _, _, err = git.NewCommand("update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { return "", err } @@ -700,13 +698,13 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head // The SHA is empty log.Warn("Empty reference, no pull head for PR #%d in %s/%s", pr.Number, g.repoOwner, g.repoName) } else { - _, _, err = git.NewCommand(g.ctx, "rev-list", "--quiet", "-1").AddDynamicArguments(pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + _, _, err = git.NewCommand("rev-list", "--quiet", "-1").AddDynamicArguments(pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { // Git update-ref remove bad references with a relative path log.Warn("Deprecated local head %s for PR #%d in %s/%s, removing %s", pr.Head.SHA, pr.Number, g.repoOwner, g.repoName, pr.GetGitRefName()) } else { // set head information - _, _, err = git.NewCommand(g.ctx, "update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(&git.RunOpts{Dir: g.repo.RepoPath()}) + _, _, err = git.NewCommand("update-ref", "--no-deref").AddDynamicArguments(pr.GetGitRefName(), pr.Head.SHA).RunStdString(ctx, &git.RunOpts{Dir: g.repo.RepoPath()}) if err != nil { log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err) } @@ -716,7 +714,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head return head, nil } -func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model.PullRequest, error) { +func (g *GiteaLocalUploader) newPullRequest(ctx context.Context, pr *base.PullRequest) (*issues_model.PullRequest, error) { var labels []*issues_model.Label for _, label := range pr.Labels { lb, ok := g.labels[label.Name] @@ -727,7 +725,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model milestoneID := g.milestones[pr.Milestone] - head, err := g.updateGitForPullRequest(pr) + head, err := g.updateGitForPullRequest(ctx, pr) if err != nil { return nil, fmt.Errorf("updateGitForPullRequest: %w", err) } @@ -767,7 +765,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model issue := issues_model.Issue{ RepoID: g.repo.ID, Repo: g.repo, - Title: prTitle, + Title: util.TruncateRunes(prTitle, 255), Index: pr.Number, Content: pr.Content, MilestoneID: milestoneID, @@ -779,7 +777,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model UpdatedUnix: timeutil.TimeStamp(pr.Updated.Unix()), } - if err := g.remapUser(pr, &issue); err != nil { + if err := g.remapUser(ctx, pr, &issue); err != nil { return nil, err } @@ -789,7 +787,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model Type: reaction.Content, CreatedUnix: timeutil.TimeStampNow(), } - if err := g.remapUser(reaction, &res); err != nil { + if err := g.remapUser(ctx, reaction, &res); err != nil { return nil, err } issue.Reactions = append(issue.Reactions, &res) @@ -839,7 +837,7 @@ func convertReviewState(state string) issues_model.ReviewType { } // CreateReviews create pull request reviews of currently migrated issues -func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { +func (g *GiteaLocalUploader) CreateReviews(ctx context.Context, reviews ...*base.Review) error { cms := make([]*issues_model.Review, 0, len(reviews)) for _, review := range reviews { var issue *issues_model.Issue @@ -860,7 +858,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()), } - if err := g.remapUser(review, &cm); err != nil { + if err := g.remapUser(ctx, review, &cm); err != nil { return err } @@ -870,7 +868,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { pr, ok := g.prCache[issue.ID] if !ok { var err error - pr, err = issues_model.GetPullRequestByIssueIDWithNoAttributes(g.ctx, issue.ID) + pr, err = issues_model.GetPullRequestByIssueIDWithNoAttributes(ctx, issue.ID) if err != nil { return err } @@ -940,7 +938,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()), } - if err := g.remapUser(review, &c); err != nil { + if err := g.remapUser(ctx, review, &c); err != nil { return err } @@ -948,7 +946,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { } } - return issues_model.InsertReviews(g.ctx, cms) + return issues_model.InsertReviews(ctx, cms) } // Rollback when migrating failed, this will rollback all the changes. @@ -962,31 +960,31 @@ func (g *GiteaLocalUploader) Rollback() error { } // Finish when migrating success, this will do some status update things. -func (g *GiteaLocalUploader) Finish() error { +func (g *GiteaLocalUploader) Finish(ctx context.Context) error { if g.repo == nil || g.repo.ID <= 0 { return ErrRepoNotCreated } // update issue_index - if err := issues_model.RecalculateIssueIndexForRepo(g.ctx, g.repo.ID); err != nil { + if err := issues_model.RecalculateIssueIndexForRepo(ctx, g.repo.ID); err != nil { return err } - if err := models.UpdateRepoStats(g.ctx, g.repo.ID); err != nil { + if err := models.UpdateRepoStats(ctx, g.repo.ID); err != nil { return err } g.repo.Status = repo_model.RepositoryReady - return repo_model.UpdateRepositoryCols(g.ctx, g.repo, "status") + return repo_model.UpdateRepositoryColsWithAutoTime(ctx, g.repo, "status") } -func (g *GiteaLocalUploader) remapUser(source user_model.ExternalUserMigrated, target user_model.ExternalUserRemappable) error { +func (g *GiteaLocalUploader) remapUser(ctx context.Context, source user_model.ExternalUserMigrated, target user_model.ExternalUserRemappable) error { var userID int64 var err error if g.sameApp { - userID, err = g.remapLocalUser(source) + userID, err = g.remapLocalUser(ctx, source) } else { - userID, err = g.remapExternalUser(source) + userID, err = g.remapExternalUser(ctx, source) } if err != nil { return err @@ -998,10 +996,10 @@ func (g *GiteaLocalUploader) remapUser(source user_model.ExternalUserMigrated, t return target.RemapExternalUser(source.GetExternalName(), source.GetExternalID(), g.doer.ID) } -func (g *GiteaLocalUploader) remapLocalUser(source user_model.ExternalUserMigrated) (int64, error) { +func (g *GiteaLocalUploader) remapLocalUser(ctx context.Context, source user_model.ExternalUserMigrated) (int64, error) { userid, ok := g.userMap[source.GetExternalID()] if !ok { - name, err := user_model.GetUserNameByID(g.ctx, source.GetExternalID()) + name, err := user_model.GetUserNameByID(ctx, source.GetExternalID()) if err != nil { return 0, err } @@ -1016,10 +1014,10 @@ func (g *GiteaLocalUploader) remapLocalUser(source user_model.ExternalUserMigrat return userid, nil } -func (g *GiteaLocalUploader) remapExternalUser(source user_model.ExternalUserMigrated) (userid int64, err error) { +func (g *GiteaLocalUploader) remapExternalUser(ctx context.Context, source user_model.ExternalUserMigrated) (userid int64, err error) { userid, ok := g.userMap[source.GetExternalID()] if !ok { - userid, err = user_model.GetUserIDByExternalUserID(g.ctx, g.gitServiceType.Name(), fmt.Sprintf("%d", source.GetExternalID())) + userid, err = user_model.GetUserIDByExternalUserID(ctx, g.gitServiceType.Name(), strconv.FormatInt(source.GetExternalID(), 10)) if err != nil { log.Error("GetUserIDByExternalUserID: %v", err) return 0, err diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index f2379dadf8..1970c0550c 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -5,7 +5,6 @@ package migrations import ( - "context" "fmt" "os" "path/filepath" @@ -40,7 +39,7 @@ func TestGiteaUploadRepo(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) var ( - ctx = context.Background() + ctx = t.Context() downloader = NewGithubDownloaderV3(ctx, "https://github.com", "", "", "", "go-xorm", "builder") repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05") uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) @@ -65,7 +64,7 @@ func TestGiteaUploadRepo(t *testing.T) { repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerID: user.ID, Name: repoName}) assert.True(t, repo.HasWiki()) - assert.EqualValues(t, repo_model.RepositoryReady, repo.Status) + assert.Equal(t, repo_model.RepositoryReady, repo.Status) milestones, err := db.Find[issues_model.Milestone](db.DefaultContext, issues_model.FindMilestoneOptions{ RepoID: repo.ID, @@ -132,8 +131,9 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) { doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + ctx := t.Context() repoName := "migrated" - uploader := NewGiteaLocalUploader(context.Background(), doer, doer.Name, repoName) + uploader := NewGiteaLocalUploader(ctx, doer, doer.Name, repoName) // call remapLocalUser uploader.sameApp = true @@ -150,9 +150,9 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) { // target := repo_model.Release{} uploader.userMap = make(map[int64]int64) - err := uploader.remapUser(&source, &target) + err := uploader.remapUser(ctx, &source, &target) assert.NoError(t, err) - assert.EqualValues(t, doer.ID, target.GetUserID()) + assert.Equal(t, doer.ID, target.GetUserID()) // // The externalID matches a known user but the name does not match, @@ -161,9 +161,9 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) { source.PublisherID = user.ID target = repo_model.Release{} uploader.userMap = make(map[int64]int64) - err = uploader.remapUser(&source, &target) + err = uploader.remapUser(ctx, &source, &target) assert.NoError(t, err) - assert.EqualValues(t, doer.ID, target.GetUserID()) + assert.Equal(t, doer.ID, target.GetUserID()) // // The externalID and externalName match an existing user, everything @@ -172,17 +172,17 @@ func TestGiteaUploadRemapLocalUser(t *testing.T) { source.PublisherName = user.Name target = repo_model.Release{} uploader.userMap = make(map[int64]int64) - err = uploader.remapUser(&source, &target) + err = uploader.remapUser(ctx, &source, &target) assert.NoError(t, err) - assert.EqualValues(t, user.ID, target.GetUserID()) + assert.Equal(t, user.ID, target.GetUserID()) } func TestGiteaUploadRemapExternalUser(t *testing.T) { unittest.PrepareTestEnv(t) doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) - + ctx := t.Context() repoName := "migrated" - uploader := NewGiteaLocalUploader(context.Background(), doer, doer.Name, repoName) + uploader := NewGiteaLocalUploader(ctx, doer, doer.Name, repoName) uploader.gitServiceType = structs.GiteaService // call remapExternalUser uploader.sameApp = false @@ -200,9 +200,9 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) { // uploader.userMap = make(map[int64]int64) target := repo_model.Release{} - err := uploader.remapUser(&source, &target) + err := uploader.remapUser(ctx, &source, &target) assert.NoError(t, err) - assert.EqualValues(t, doer.ID, target.GetUserID()) + assert.Equal(t, doer.ID, target.GetUserID()) // // Link the external ID to an existing user @@ -223,9 +223,9 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) { // uploader.userMap = make(map[int64]int64) target = repo_model.Release{} - err = uploader.remapUser(&source, &target) + err = uploader.remapUser(ctx, &source, &target) assert.NoError(t, err) - assert.EqualValues(t, linkedUser.ID, target.GetUserID()) + assert.Equal(t, linkedUser.ID, target.GetUserID()) } func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { @@ -237,9 +237,9 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { fromRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) baseRef := "master" assert.NoError(t, git.InitRepository(git.DefaultContext, fromRepo.RepoPath(), false, fromRepo.ObjectFormatName)) - err := git.NewCommand(git.DefaultContext, "symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseRef).Run(&git.RunOpts{Dir: fromRepo.RepoPath()}) + err := git.NewCommand("symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseRef).Run(git.DefaultContext, &git.RunOpts{Dir: fromRepo.RepoPath()}) assert.NoError(t, err) - assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# Testing Repository\n\nOriginally created in: %s", fromRepo.RepoPath())), 0o644)) + assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte("# Testing Repository\n\nOriginally created in: "+fromRepo.RepoPath()), 0o644)) assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true)) signature := git.Signature{ Email: "test@example.com", @@ -261,7 +261,7 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { // fromRepo branch1 // headRef := "branch1" - _, _, err = git.NewCommand(git.DefaultContext, "checkout", "-b").AddDynamicArguments(headRef).RunStdString(&git.RunOpts{Dir: fromRepo.RepoPath()}) + _, _, err = git.NewCommand("checkout", "-b").AddDynamicArguments(headRef).RunStdString(git.DefaultContext, &git.RunOpts{Dir: fromRepo.RepoPath()}) assert.NoError(t, err) assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte("SOMETHING"), 0o644)) assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true)) @@ -285,9 +285,9 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { assert.NoError(t, git.CloneWithArgs(git.DefaultContext, nil, fromRepo.RepoPath(), forkRepo.RepoPath(), git.CloneRepoOptions{ Branch: headRef, })) - _, _, err = git.NewCommand(git.DefaultContext, "checkout", "-b").AddDynamicArguments(forkHeadRef).RunStdString(&git.RunOpts{Dir: forkRepo.RepoPath()}) + _, _, err = git.NewCommand("checkout", "-b").AddDynamicArguments(forkHeadRef).RunStdString(git.DefaultContext, &git.RunOpts{Dir: forkRepo.RepoPath()}) assert.NoError(t, err) - assert.NoError(t, os.WriteFile(filepath.Join(forkRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# branch2 %s", forkRepo.RepoPath())), 0o644)) + assert.NoError(t, os.WriteFile(filepath.Join(forkRepo.RepoPath(), "README.md"), []byte("# branch2 "+forkRepo.RepoPath()), 0o644)) assert.NoError(t, git.AddChanges(forkRepo.RepoPath(), true)) assert.NoError(t, git.CommitChanges(forkRepo.RepoPath(), git.CommitChangesOptions{ Committer: &signature, @@ -301,11 +301,12 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { assert.NoError(t, err) toRepoName := "migrated" - uploader := NewGiteaLocalUploader(context.Background(), fromRepoOwner, fromRepoOwner.Name, toRepoName) + ctx := t.Context() + uploader := NewGiteaLocalUploader(ctx, fromRepoOwner, fromRepoOwner.Name, toRepoName) uploader.gitServiceType = structs.GiteaService - assert.NoError(t, repo_service.Init(context.Background())) - assert.NoError(t, uploader.CreateRepo(&base.Repository{ + assert.NoError(t, repo_service.Init(t.Context())) + assert.NoError(t, uploader.CreateRepo(ctx, &base.Repository{ Description: "description", OriginalURL: fromRepo.RepoPath(), CloneURL: fromRepo.RepoPath(), @@ -505,16 +506,16 @@ func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { testCase.pr.EnsuredSafe = true - head, err := uploader.updateGitForPullRequest(&testCase.pr) + head, err := uploader.updateGitForPullRequest(ctx, &testCase.pr) assert.NoError(t, err) - assert.EqualValues(t, testCase.head, head) + assert.Equal(t, testCase.head, head) log.Info(stopMark) logFiltered, logStopped := logChecker.Check(5 * time.Second) assert.True(t, logStopped) if len(testCase.logFilter) > 0 { - assert.EqualValues(t, testCase.logFiltered, logFiltered, "for log message filters: %v", testCase.logFilter) + assert.Equal(t, testCase.logFiltered, logFiltered, "for log message filters: %v", testCase.logFilter) } }) } diff --git a/services/migrations/github.go b/services/migrations/github.go index 604ab84b39..2ce11615c6 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -20,7 +20,7 @@ import ( "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/structs" - "github.com/google/go-github/v61/github" + "github.com/google/go-github/v71/github" "golang.org/x/oauth2" ) @@ -64,7 +64,6 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType { // from github via APIv3 type GithubDownloaderV3 struct { base.NullDownloader - ctx context.Context clients []*github.Client baseURL string repoOwner string @@ -79,20 +78,19 @@ type GithubDownloaderV3 struct { } // NewGithubDownloaderV3 creates a github Downloader via github v3 API -func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 { +func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 { downloader := GithubDownloaderV3{ userName: userName, baseURL: baseURL, password: password, - ctx: ctx, repoOwner: repoOwner, repoName: repoName, maxPerPage: 100, } if token != "" { - tokens := strings.Split(token, ",") - for _, token := range tokens { + tokens := strings.SplitSeq(token, ",") + for token := range tokens { token = strings.TrimSpace(token) ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, @@ -135,18 +133,13 @@ func (g *GithubDownloaderV3) LogString() string { func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) { githubClient := github.NewClient(client) if baseURL != "https://github.com" { - githubClient, _ = github.NewClient(client).WithEnterpriseURLs(baseURL, baseURL) + githubClient, _ = githubClient.WithEnterpriseURLs(baseURL, baseURL) } g.clients = append(g.clients, githubClient) g.rates = append(g.rates, nil) } -// SetContext set context -func (g *GithubDownloaderV3) SetContext(ctx context.Context) { - g.ctx = ctx -} - -func (g *GithubDownloaderV3) waitAndPickClient() { +func (g *GithubDownloaderV3) waitAndPickClient(ctx context.Context) { var recentIdx int var maxRemaining int for i := 0; i < len(g.clients); i++ { @@ -160,13 +153,13 @@ func (g *GithubDownloaderV3) waitAndPickClient() { for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining { timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time)) select { - case <-g.ctx.Done(): + case <-ctx.Done(): timer.Stop() return case <-timer.C: } - err := g.RefreshRate() + err := g.RefreshRate(ctx) if err != nil { log.Error("g.getClient().RateLimit.Get: %s", err) } @@ -174,8 +167,8 @@ func (g *GithubDownloaderV3) waitAndPickClient() { } // RefreshRate update the current rate (doesn't count in rate limit) -func (g *GithubDownloaderV3) RefreshRate() error { - rates, _, err := g.getClient().RateLimit.Get(g.ctx) +func (g *GithubDownloaderV3) RefreshRate(ctx context.Context) error { + rates, _, err := g.getClient().RateLimit.Get(ctx) if err != nil { // if rate limit is not enabled, ignore it if strings.Contains(err.Error(), "404") { @@ -198,9 +191,9 @@ func (g *GithubDownloaderV3) setRate(rate *github.Rate) { } // GetRepoInfo returns a repository information -func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { - g.waitAndPickClient() - gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName) +func (g *GithubDownloaderV3) GetRepoInfo(ctx context.Context) (*base.Repository, error) { + g.waitAndPickClient(ctx) + gr, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName) if err != nil { return nil, err } @@ -219,9 +212,9 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) { } // GetTopics return github topics -func (g *GithubDownloaderV3) GetTopics() ([]string, error) { - g.waitAndPickClient() - r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName) +func (g *GithubDownloaderV3) GetTopics(ctx context.Context) ([]string, error) { + g.waitAndPickClient(ctx) + r, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName) if err != nil { return nil, err } @@ -230,12 +223,12 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) { } // GetMilestones returns milestones -func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { +func (g *GithubDownloaderV3) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { perPage := g.maxPerPage milestones := make([]*base.Milestone, 0, perPage) for i := 1; ; i++ { - g.waitAndPickClient() - ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName, + g.waitAndPickClient(ctx) + ms, resp, err := g.getClient().Issues.ListMilestones(ctx, g.repoOwner, g.repoName, &github.MilestoneListOptions{ State: "all", ListOptions: github.ListOptions{ @@ -279,12 +272,12 @@ func convertGithubLabel(label *github.Label) *base.Label { } // GetLabels returns labels -func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { +func (g *GithubDownloaderV3) GetLabels(ctx context.Context) ([]*base.Label, error) { perPage := g.maxPerPage labels := make([]*base.Label, 0, perPage) for i := 1; ; i++ { - g.waitAndPickClient() - ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName, + g.waitAndPickClient(ctx) + ls, resp, err := g.getClient().Issues.ListLabels(ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, PerPage: perPage, @@ -304,7 +297,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { return labels, nil } -func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release { +func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *github.RepositoryRelease) *base.Release { // GitHub allows commitish to be a reference. // In this case, we need to remove the prefix, i.e. convert "refs/heads/main" to "main". targetCommitish := strings.TrimPrefix(rel.GetTargetCommitish(), git.BranchPrefix) @@ -339,12 +332,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) Created: asset.CreatedAt.Time, Updated: asset.UpdatedAt.Time, DownloadFunc: func() (io.ReadCloser, error) { - g.waitAndPickClient() - readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil) + g.waitAndPickClient(ctx) + readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(ctx, g.repoOwner, g.repoName, assetID, nil) if err != nil { return nil, err } - if err := g.RefreshRate(); err != nil { + if err := g.RefreshRate(ctx); err != nil { log.Error("g.getClient().RateLimits: %s", err) } @@ -364,13 +357,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) return io.NopCloser(strings.NewReader(redirectURL)), nil } - g.waitAndPickClient() - req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil) + g.waitAndPickClient(ctx) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, redirectURL, nil) if err != nil { return nil, err } resp, err := httpClient.Do(req) - err1 := g.RefreshRate() + err1 := g.RefreshRate(ctx) if err1 != nil { log.Error("g.RefreshRate(): %s", err1) } @@ -385,12 +378,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) } // GetReleases returns releases -func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { +func (g *GithubDownloaderV3) GetReleases(ctx context.Context) ([]*base.Release, error) { perPage := g.maxPerPage releases := make([]*base.Release, 0, perPage) for i := 1; ; i++ { - g.waitAndPickClient() - ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName, + g.waitAndPickClient(ctx) + ls, resp, err := g.getClient().Repositories.ListReleases(ctx, g.repoOwner, g.repoName, &github.ListOptions{ Page: i, PerPage: perPage, @@ -401,7 +394,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { g.setRate(&resp.Rate) for _, release := range ls { - releases = append(releases, g.convertGithubRelease(release)) + releases = append(releases, g.convertGithubRelease(ctx, release)) } if len(ls) < perPage { break @@ -411,7 +404,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { } // GetIssues returns issues according start and limit -func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -426,8 +419,8 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, } allIssues := make([]*base.Issue, 0, perPage) - g.waitAndPickClient() - issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) + g.waitAndPickClient(ctx) + issues, resp, err := g.getClient().Issues.ListByRepo(ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } @@ -447,10 +440,12 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, var reactions []*base.Reaction if !g.SkipReactions { for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{ - Page: i, - PerPage: perPage, + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: perPage, + }, }) if err != nil { return nil, false, err @@ -503,12 +498,12 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool { } // GetComments returns comments according issueNumber -func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - comments, err := g.getComments(commentable) +func (g *GithubDownloaderV3) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { + comments, err := g.getComments(ctx, commentable) return comments, false, err } -func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) { +func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -522,8 +517,8 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. }, } for { - g.waitAndPickClient() - comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt) + g.waitAndPickClient(ctx) + comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } @@ -533,10 +528,12 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. var reactions []*base.Reaction if !g.SkipReactions { for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ - Page: i, - PerPage: g.maxPerPage, + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: g.maxPerPage, + }, }) if err != nil { return nil, err @@ -576,7 +573,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. } // GetAllComments returns repository comments according page and perPageSize -func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) { +func (g *GithubDownloaderV3) GetAllComments(ctx context.Context, page, perPage int) ([]*base.Comment, bool, error) { var ( allComments = make([]*base.Comment, 0, perPage) created = "created" @@ -594,8 +591,8 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, }, } - g.waitAndPickClient() - comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt) + g.waitAndPickClient(ctx) + comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, 0, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } @@ -608,10 +605,12 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, var reactions []*base.Reaction if !g.SkipReactions { for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ - Page: i, - PerPage: g.maxPerPage, + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: g.maxPerPage, + }, }) if err != nil { return nil, false, err @@ -648,7 +647,7 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, } // GetPullRequests returns pull requests according page and perPage -func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -662,8 +661,8 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq }, } allPRs := make([]*base.PullRequest, 0, perPage) - g.waitAndPickClient() - prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) + g.waitAndPickClient(ctx) + prs, resp, err := g.getClient().PullRequests.List(ctx, g.repoOwner, g.repoName, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } @@ -679,10 +678,12 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq var reactions []*base.Reaction if !g.SkipReactions { for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{ - Page: i, - PerPage: perPage, + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: perPage, + }, }) if err != nil { return nil, false, err @@ -702,7 +703,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq } // download patch and saved as tmp file - g.waitAndPickClient() + g.waitAndPickClient(ctx) allPRs = append(allPRs, &base.PullRequest{ Title: pr.GetTitle(), @@ -759,17 +760,19 @@ func convertGithubReview(r *github.PullRequestReview) *base.Review { } } -func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullRequestComment) ([]*base.ReviewComment, error) { +func (g *GithubDownloaderV3) convertGithubReviewComments(ctx context.Context, cs []*github.PullRequestComment) ([]*base.ReviewComment, error) { rcs := make([]*base.ReviewComment, 0, len(cs)) for _, c := range cs { // get reactions var reactions []*base.Reaction if !g.SkipReactions { for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{ - Page: i, - PerPage: g.maxPerPage, + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: g.maxPerPage, + }, }) if err != nil { return nil, err @@ -806,7 +809,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques } // GetReviews returns pull requests review -func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { allReviews := make([]*base.Review, 0, g.maxPerPage) if g.SkipReviews { return allReviews, nil @@ -816,8 +819,8 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } // Get approve/request change reviews for { - g.waitAndPickClient() - reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) + g.waitAndPickClient(ctx) + reviews, resp, err := g.getClient().PullRequests.ListReviews(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } @@ -830,14 +833,14 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev PerPage: g.maxPerPage, } for { - g.waitAndPickClient() - reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2) + g.waitAndPickClient(ctx) + reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } g.setRate(&resp.Rate) - cs, err := g.convertGithubReviewComments(reviewComments) + cs, err := g.convertGithubReviewComments(ctx, reviewComments) if err != nil { return nil, err } @@ -856,8 +859,8 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } // Get requested reviews for { - g.waitAndPickClient() - reviewers, resp, err := g.getClient().PullRequests.ListReviewers(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) + g.waitAndPickClient(ctx) + reviewers, resp, err := g.getClient().PullRequests.ListReviewers(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %w", err) } @@ -879,3 +882,18 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } return allReviews, nil } + +// FormatCloneURL add authentication into remote URLs +func (g *GithubDownloaderV3) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) { + u, err := url.Parse(remoteAddr) + if err != nil { + return "", err + } + if len(opts.AuthToken) > 0 { + // "multiple tokens" are used to benefit more "API rate limit quota" + // git clone doesn't count for rate limits, so only use the first token. + // source: https://github.com/orgs/community/discussions/44515 + u.User = url.UserPassword("oauth2", strings.Split(opts.AuthToken, ",")[0]) + } + return u.String(), nil +} diff --git a/services/migrations/github_test.go b/services/migrations/github_test.go index 2b89e6dc0f..6d1a5378b9 100644 --- a/services/migrations/github_test.go +++ b/services/migrations/github_test.go @@ -5,7 +5,6 @@ package migrations import ( - "context" "os" "testing" "time" @@ -13,6 +12,7 @@ import ( base "code.gitea.io/gitea/modules/migration" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGitHubDownloadRepo(t *testing.T) { @@ -21,11 +21,12 @@ func TestGitHubDownloadRepo(t *testing.T) { if token == "" { t.Skip("Skipping GitHub migration test because GITHUB_READ_TOKEN is empty") } - downloader := NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", token, "go-gitea", "test_repo") - err := downloader.RefreshRate() + ctx := t.Context() + downloader := NewGithubDownloaderV3(ctx, "https://github.com", "", "", token, "go-gitea", "test_repo") + err := downloader.RefreshRate(ctx) assert.NoError(t, err) - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) assertRepositoryEqual(t, &base.Repository{ Name: "test_repo", @@ -36,11 +37,11 @@ func TestGitHubDownloadRepo(t *testing.T) { DefaultBranch: "master", }, repo) - topics, err := downloader.GetTopics() + topics, err := downloader.GetTopics(ctx) assert.NoError(t, err) assert.Contains(t, topics, "gitea") - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) assertMilestonesEqual(t, []*base.Milestone{ { @@ -63,7 +64,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, milestones) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assertLabelsEqual(t, []*base.Label{ { @@ -113,7 +114,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, labels) - releases, err := downloader.GetReleases() + releases, err := downloader.GetReleases(ctx) assert.NoError(t, err) assertReleasesEqual(t, []*base.Release{ { @@ -129,7 +130,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, releases) // downloader.GetIssues() - issues, isEnd, err := downloader.GetIssues(1, 2) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 2) assert.NoError(t, err) assert.False(t, isEnd) assertIssuesEqual(t, []*base.Issue{ @@ -218,7 +219,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(&base.Issue{Number: 2, ForeignIndex: 2}) + comments, _, err := downloader.GetComments(ctx, &base.Issue{Number: 2, ForeignIndex: 2}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -248,7 +249,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, comments) // downloader.GetPullRequests() - prs, _, err := downloader.GetPullRequests(1, 2) + prs, _, err := downloader.GetPullRequests(ctx, 1, 2) assert.NoError(t, err) assertPullRequestsEqual(t, []*base.PullRequest{ { @@ -338,7 +339,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, prs) - reviews, err := downloader.GetReviews(&base.PullRequest{Number: 3, ForeignIndex: 3}) + reviews, err := downloader.GetReviews(ctx, &base.PullRequest{Number: 3, ForeignIndex: 3}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -370,7 +371,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, err = downloader.GetReviews(&base.PullRequest{Number: 4, ForeignIndex: 4}) + reviews, err = downloader.GetReviews(ctx, &base.PullRequest{Number: 4, ForeignIndex: 4}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -429,3 +430,36 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) } + +func TestGithubMultiToken(t *testing.T) { + testCases := []struct { + desc string + token string + expectedCloneURL string + }{ + { + desc: "Single Token", + token: "single_token", + expectedCloneURL: "https://oauth2:single_token@github.com", + }, + { + desc: "Multi Token", + token: "token1,token2", + expectedCloneURL: "https://oauth2:token1@github.com", + }, + } + factory := GithubDownloaderV3Factory{} + + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + opts := base.MigrateOptions{CloneAddr: "https://github.com/go-gitea/gitea", AuthToken: tC.token} + client, err := factory.New(t.Context(), opts) + require.NoError(t, err) + + cloneURL, err := client.FormatCloneURL(opts, "https://github.com") + require.NoError(t, err) + + assert.Equal(t, tC.expectedCloneURL, cloneURL) + }) + } +} diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index 07d5040b5b..a19a04bc44 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -16,6 +16,7 @@ import ( "time" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" @@ -80,7 +81,6 @@ func (r *gitlabIIDResolver) generatePullRequestNumber(mrIID int) int64 { // because Gitlab has individual Issue and Pull Request numbers. type GitlabDownloader struct { base.NullDownloader - ctx context.Context client *gitlab.Client baseURL string repoID int @@ -143,7 +143,6 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw } return &GitlabDownloader{ - ctx: ctx, client: gitlabClient, baseURL: baseURL, repoID: gr.ID, @@ -164,14 +163,9 @@ func (g *GitlabDownloader) LogString() string { return fmt.Sprintf("<GitlabDownloader %s [%d]/%s>", g.baseURL, g.repoID, g.repoName) } -// SetContext set context -func (g *GitlabDownloader) SetContext(ctx context.Context) { - g.ctx = ctx -} - // GetRepoInfo returns a repository information -func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) { - gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { + gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(ctx)) if err != nil { return nil, err } @@ -207,8 +201,8 @@ func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) { } // GetTopics return gitlab topics -func (g *GitlabDownloader) GetTopics() ([]string, error) { - gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetTopics(ctx context.Context) ([]string, error) { + gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(ctx)) if err != nil { return nil, err } @@ -216,7 +210,7 @@ func (g *GitlabDownloader) GetTopics() ([]string, error) { } // GetMilestones returns milestones -func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) { +func (g *GitlabDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { perPage := g.maxPerPage state := "all" milestones := make([]*base.Milestone, 0, perPage) @@ -227,7 +221,7 @@ func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) { Page: i, PerPage: perPage, }, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) if err != nil { return nil, err } @@ -288,14 +282,14 @@ func (g *GitlabDownloader) normalizeColor(val string) string { } // GetLabels returns labels -func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) { +func (g *GitlabDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) { perPage := g.maxPerPage labels := make([]*base.Label, 0, perPage) for i := 1; ; i++ { ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{ Page: i, PerPage: perPage, - }}, nil, gitlab.WithContext(g.ctx)) + }}, nil, gitlab.WithContext(ctx)) if err != nil { return nil, err } @@ -314,7 +308,7 @@ func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) { return labels, nil } -func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release { +func (g *GitlabDownloader) convertGitlabRelease(ctx context.Context, rel *gitlab.Release) *base.Release { var zero int r := &base.Release{ TagName: rel.TagName, @@ -337,7 +331,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea Size: &zero, DownloadCount: &zero, DownloadFunc: func() (io.ReadCloser, error) { - link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, assetID, gitlab.WithContext(g.ctx)) + link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, assetID, gitlab.WithContext(ctx)) if err != nil { return nil, err } @@ -347,11 +341,11 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea return io.NopCloser(strings.NewReader(link.URL)), nil } - req, err := http.NewRequest("GET", link.URL, nil) + req, err := http.NewRequest(http.MethodGet, link.URL, nil) if err != nil { return nil, err } - req = req.WithContext(g.ctx) + req = req.WithContext(ctx) resp, err := httpClient.Do(req) if err != nil { return nil, err @@ -366,7 +360,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea } // GetReleases returns releases -func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { +func (g *GitlabDownloader) GetReleases(ctx context.Context) ([]*base.Release, error) { perPage := g.maxPerPage releases := make([]*base.Release, 0, perPage) for i := 1; ; i++ { @@ -375,13 +369,13 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { Page: i, PerPage: perPage, }, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) if err != nil { return nil, err } for _, release := range ls { - releases = append(releases, g.convertGitlabRelease(release)) + releases = append(releases, g.convertGitlabRelease(ctx, release)) } if len(ls) < perPage { break @@ -397,7 +391,7 @@ type gitlabIssueContext struct { // GetIssues returns issues according start and limit // // Note: issue label description and colors are not supported by the go-gitlab library at this time -func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (g *GitlabDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) { state := "all" sort := "asc" @@ -416,7 +410,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er allIssues := make([]*base.Issue, 0, perPage) - issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(g.ctx)) + issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(ctx)) if err != nil { return nil, false, fmt.Errorf("error while listing issues: %w", err) } @@ -436,7 +430,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er var reactions []*gitlab.AwardEmoji awardPage := 1 for { - awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx)) + awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(ctx)) if err != nil { return nil, false, fmt.Errorf("error while listing issue awards: %w", err) } @@ -477,7 +471,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er // GetComments returns comments according issueNumber // TODO: figure out how to transfer comment reactions -func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (g *GitlabDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { context, ok := commentable.GetContext().(gitlabIssueContext) if !ok { return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) @@ -495,12 +489,12 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListIssueDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) } else { comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListMergeRequestDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) } if err != nil { @@ -528,25 +522,29 @@ func (g *GitlabDownloader) GetComments(commentable base.Commentable) ([]*base.Co Page: page, PerPage: g.maxPerPage, }, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) } else { stateEvents, resp, err = g.client.ResourceStateEvents.ListIssueStateEvents(g.repoID, int(commentable.GetForeignIndex()), &gitlab.ListStateEventsOptions{ ListOptions: gitlab.ListOptions{ Page: page, PerPage: g.maxPerPage, }, - }, nil, gitlab.WithContext(g.ctx)) + }, nil, gitlab.WithContext(ctx)) } if err != nil { return nil, false, fmt.Errorf("error while listing state events: %v %w", g.repoID, err) } for _, stateEvent := range stateEvents { + posterUserID, posterUsername := user.GhostUserID, user.GhostUserName + if stateEvent.User != nil { + posterUserID, posterUsername = int64(stateEvent.User.ID), stateEvent.User.Username + } comment := &base.Comment{ IssueIndex: commentable.GetLocalIndex(), Index: int64(stateEvent.ID), - PosterID: int64(stateEvent.User.ID), - PosterName: stateEvent.User.Username, + PosterID: posterUserID, + PosterName: posterUsername, Content: "", Created: *stateEvent.CreatedAt, } @@ -604,7 +602,7 @@ func (g *GitlabDownloader) convertNoteToComment(localIndex int64, note *gitlab.N } // GetPullRequests returns pull requests according page and perPage -func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (g *GitlabDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -620,7 +618,7 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque allPRs := make([]*base.PullRequest, 0, perPage) - prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(g.ctx)) + prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(ctx)) if err != nil { return nil, false, fmt.Errorf("error while listing merge requests: %w", err) } @@ -673,7 +671,7 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque var reactions []*gitlab.AwardEmoji awardPage := 1 for { - awards, _, err := g.client.AwardEmoji.ListMergeRequestAwardEmoji(g.repoID, pr.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx)) + awards, _, err := g.client.AwardEmoji.ListMergeRequestAwardEmoji(g.repoID, pr.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(ctx)) if err != nil { return nil, false, fmt.Errorf("error while listing merge requests awards: %w", err) } @@ -733,8 +731,8 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests review -func (g *GitlabDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { - approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignIndex()), gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { + approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(reviewable.GetForeignIndex()), gitlab.WithContext(ctx)) if err != nil { if resp != nil && resp.StatusCode == http.StatusNotFound { log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) diff --git a/services/migrations/gitlab_test.go b/services/migrations/gitlab_test.go index 556fe771c5..73a1b6a276 100644 --- a/services/migrations/gitlab_test.go +++ b/services/migrations/gitlab_test.go @@ -4,7 +4,6 @@ package migrations import ( - "context" "fmt" "net/http" "net/http/httptest" @@ -31,12 +30,12 @@ func TestGitlabDownloadRepo(t *testing.T) { if err != nil || resp.StatusCode != http.StatusOK { t.Skipf("Can't access test repo, skipping %s", t.Name()) } - - downloader, err := NewGitlabDownloader(context.Background(), "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken) + ctx := t.Context() + downloader, err := NewGitlabDownloader(ctx, "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken) if err != nil { t.Fatalf("NewGitlabDownloader is nil: %v", err) } - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) // Repo Owner is blank in Gitlab Group repos assertRepositoryEqual(t, &base.Repository{ @@ -48,12 +47,12 @@ func TestGitlabDownloadRepo(t *testing.T) { DefaultBranch: "master", }, repo) - topics, err := downloader.GetTopics() + topics, err := downloader.GetTopics(ctx) assert.NoError(t, err) assert.Len(t, topics, 2) - assert.EqualValues(t, []string{"migration", "test"}, topics) + assert.Equal(t, []string{"migration", "test"}, topics) - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) assertMilestonesEqual(t, []*base.Milestone{ { @@ -71,7 +70,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, milestones) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assertLabelsEqual(t, []*base.Label{ { @@ -112,7 +111,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, labels) - releases, err := downloader.GetReleases() + releases, err := downloader.GetReleases(ctx) assert.NoError(t, err) assertReleasesEqual(t, []*base.Release{ { @@ -126,7 +125,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, releases) - issues, isEnd, err := downloader.GetIssues(1, 2) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 2) assert.NoError(t, err) assert.False(t, isEnd) @@ -214,7 +213,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{ + comments, _, err := downloader.GetComments(ctx, &base.Issue{ Number: 2, ForeignIndex: 2, Context: gitlabIssueContext{IsMergeRequest: false}, @@ -255,7 +254,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, comments) - prs, _, err := downloader.GetPullRequests(1, 1) + prs, _, err := downloader.GetPullRequests(ctx, 1, 1) assert.NoError(t, err) assertPullRequestsEqual(t, []*base.PullRequest{ { @@ -304,7 +303,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(&base.PullRequest{Number: 1, ForeignIndex: 1}) + rvs, err := downloader.GetReviews(ctx, &base.PullRequest{Number: 1, ForeignIndex: 1}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -323,7 +322,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, rvs) - rvs, err = downloader.GetReviews(&base.PullRequest{Number: 2, ForeignIndex: 2}) + rvs, err = downloader.GetReviews(ctx, &base.PullRequest{Number: 2, ForeignIndex: 2}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { @@ -423,9 +422,8 @@ func TestGitlabGetReviews(t *testing.T) { defer gitlabClientMockTeardown(server) repoID := 1324 - + ctx := t.Context() downloader := &GitlabDownloader{ - ctx: context.Background(), client: client, repoID: repoID, } @@ -465,7 +463,7 @@ func TestGitlabGetReviews(t *testing.T) { mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/%d/approvals", testCase.repoID, testCase.prID), mock) id := int64(testCase.prID) - rvs, err := downloader.GetReviews(&base.Issue{Number: id, ForeignIndex: id}) + rvs, err := downloader.GetReviews(ctx, &base.Issue{Number: id, ForeignIndex: id}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{&review}, rvs) } @@ -503,7 +501,7 @@ func TestAwardsToReactions(t *testing.T) { assert.NoError(t, json.Unmarshal([]byte(testResponse), &awards)) reactions := downloader.awardsToReactions(awards) - assert.EqualValues(t, []*base.Reaction{ + assert.Equal(t, []*base.Reaction{ { UserName: "lafriks", UserID: 1241334, @@ -595,7 +593,7 @@ func TestNoteToComment(t *testing.T) { for i, note := range notes { actualComment := *downloader.convertNoteToComment(17, ¬e) - assert.EqualValues(t, actualComment, comments[i]) + assert.Equal(t, actualComment, comments[i]) } } diff --git a/services/migrations/gogs.go b/services/migrations/gogs.go index 72c52d180b..a4f84dbf72 100644 --- a/services/migrations/gogs.go +++ b/services/migrations/gogs.go @@ -13,7 +13,6 @@ import ( "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" - "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/structs" "github.com/gogs/go-gogs-client" @@ -60,16 +59,14 @@ func (f *GogsDownloaderFactory) GitServiceType() structs.GitServiceType { // from gogs via API type GogsDownloader struct { base.NullDownloader - ctx context.Context - client *gogs.Client baseURL string repoOwner string repoName string userName string password string + token string openIssuesFinished bool openIssuesPages int - transport http.RoundTripper } // String implements Stringer @@ -84,53 +81,45 @@ func (g *GogsDownloader) LogString() string { return fmt.Sprintf("<GogsDownloader %s %s/%s>", g.baseURL, g.repoOwner, g.repoName) } -// SetContext set context -func (g *GogsDownloader) SetContext(ctx context.Context) { - g.ctx = ctx -} - // NewGogsDownloader creates a gogs Downloader via gogs API -func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader { +func NewGogsDownloader(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader { downloader := GogsDownloader{ - ctx: ctx, baseURL: baseURL, userName: userName, password: password, + token: token, repoOwner: repoOwner, repoName: repoName, } + return &downloader +} - var client *gogs.Client - if len(token) != 0 { - client = gogs.NewClient(baseURL, token) - downloader.userName = token - } else { - transport := NewMigrationHTTPTransport() - transport.Proxy = func(req *http.Request) (*url.URL, error) { - req.SetBasicAuth(userName, password) - return proxy.Proxy()(req) - } - downloader.transport = transport - - client = gogs.NewClient(baseURL, "") - client.SetHTTPClient(&http.Client{ - Transport: &downloader, - }) - } +type roundTripperFunc func(req *http.Request) (*http.Response, error) - downloader.client = client - return &downloader +func (rt roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return rt(r) } -// RoundTrip wraps the provided request within this downloader's context and passes it to our internal http.Transport. -// This implements http.RoundTripper and makes the gogs client requests cancellable even though it is not cancellable itself -func (g *GogsDownloader) RoundTrip(req *http.Request) (*http.Response, error) { - return g.transport.RoundTrip(req.WithContext(g.ctx)) +func (g *GogsDownloader) client(ctx context.Context) *gogs.Client { + // Gogs client lacks the context support, so we use a custom transport + // Then each request uses a dedicated client with its own context + httpTransport := NewMigrationHTTPTransport() + gogsClient := gogs.NewClient(g.baseURL, g.token) + gogsClient.SetHTTPClient(&http.Client{ + Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) { + if g.password != "" { + // Gogs client lacks the support for basic auth, this is the only way to set it + req.SetBasicAuth(g.userName, g.password) + } + return httpTransport.RoundTrip(req.WithContext(ctx)) + }), + }) + return gogsClient } // GetRepoInfo returns a repository information -func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) { - gr, err := g.client.GetRepo(g.repoOwner, g.repoName) +func (g *GogsDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { + gr, err := g.client(ctx).GetRepo(g.repoOwner, g.repoName) if err != nil { return nil, err } @@ -148,11 +137,11 @@ func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) { } // GetMilestones returns milestones -func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) { +func (g *GogsDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { perPage := 100 milestones := make([]*base.Milestone, 0, perPage) - ms, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName) + ms, err := g.client(ctx).ListRepoMilestones(g.repoOwner, g.repoName) if err != nil { return nil, err } @@ -171,10 +160,10 @@ func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) { } // GetLabels returns labels -func (g *GogsDownloader) GetLabels() ([]*base.Label, error) { +func (g *GogsDownloader) GetLabels(ctx context.Context) ([]*base.Label, error) { perPage := 100 labels := make([]*base.Label, 0, perPage) - ls, err := g.client.ListRepoLabels(g.repoOwner, g.repoName) + ls, err := g.client(ctx).ListRepoLabels(g.repoOwner, g.repoName) if err != nil { return nil, err } @@ -187,7 +176,7 @@ func (g *GogsDownloader) GetLabels() ([]*base.Label, error) { } // GetIssues returns issues according start and limit, perPage is not supported -func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) { +func (g *GogsDownloader) GetIssues(ctx context.Context, page, _ int) ([]*base.Issue, bool, error) { var state string if g.openIssuesFinished { state = string(gogs.STATE_CLOSED) @@ -197,7 +186,7 @@ func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) { g.openIssuesPages = page } - issues, isEnd, err := g.getIssues(page, state) + issues, isEnd, err := g.getIssues(ctx, page, state) if err != nil { return nil, false, err } @@ -212,10 +201,10 @@ func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) { return issues, false, nil } -func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, error) { +func (g *GogsDownloader) getIssues(ctx context.Context, page int, state string) ([]*base.Issue, bool, error) { allIssues := make([]*base.Issue, 0, 10) - issues, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{ + issues, err := g.client(ctx).ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{ Page: page, State: state, }) @@ -234,10 +223,10 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (g *GogsDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { allComments := make([]*base.Comment, 0, 100) - comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex()) + comments, err := g.client(ctx).ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex()) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %w", err) } @@ -261,7 +250,7 @@ func (g *GogsDownloader) GetComments(commentable base.Commentable) ([]*base.Comm } // GetTopics return repository topics -func (g *GogsDownloader) GetTopics() ([]string, error) { +func (g *GogsDownloader) GetTopics(_ context.Context) ([]string, error) { return []string{}, nil } diff --git a/services/migrations/gogs_test.go b/services/migrations/gogs_test.go index 610af183de..503b669f8e 100644 --- a/services/migrations/gogs_test.go +++ b/services/migrations/gogs_test.go @@ -4,7 +4,6 @@ package migrations import ( - "context" "net/http" "os" "testing" @@ -28,9 +27,9 @@ func TestGogsDownloadRepo(t *testing.T) { t.Skipf("visit test repo failed, ignored") return } - - downloader := NewGogsDownloader(context.Background(), "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO") - repo, err := downloader.GetRepoInfo() + ctx := t.Context() + downloader := NewGogsDownloader(ctx, "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO") + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) assertRepositoryEqual(t, &base.Repository{ @@ -42,7 +41,7 @@ func TestGogsDownloadRepo(t *testing.T) { DefaultBranch: "master", }, repo) - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) assertMilestonesEqual(t, []*base.Milestone{ { @@ -51,7 +50,7 @@ func TestGogsDownloadRepo(t *testing.T) { }, }, milestones) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assertLabelsEqual(t, []*base.Label{ { @@ -85,7 +84,7 @@ func TestGogsDownloadRepo(t *testing.T) { }, labels) // downloader.GetIssues() - issues, isEnd, err := downloader.GetIssues(1, 8) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 8) assert.NoError(t, err) assert.False(t, isEnd) assertIssuesEqual(t, []*base.Issue{ @@ -110,7 +109,7 @@ func TestGogsDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, _, err := downloader.GetComments(&base.Issue{Number: 1, ForeignIndex: 1}) + comments, _, err := downloader.GetComments(ctx, &base.Issue{Number: 1, ForeignIndex: 1}) assert.NoError(t, err) assertCommentsEqual(t, []*base.Comment{ { @@ -134,6 +133,6 @@ func TestGogsDownloadRepo(t *testing.T) { }, comments) // downloader.GetPullRequests() - _, _, err = downloader.GetPullRequests(1, 3) + _, _, err = downloader.GetPullRequests(ctx, 1, 3) assert.Error(t, err) } diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 51b22d6111..eba9c79df5 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -6,6 +6,7 @@ package migrations import ( "context" + "errors" "fmt" "net" "net/url" @@ -74,11 +75,9 @@ func IsMigrateURLAllowed(remoteURL string, doer *user_model.User) error { return &git.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true} } - hostName, _, err := net.SplitHostPort(u.Host) - if err != nil { - // u.Host can be "host" or "host:port" - err = nil //nolint - hostName = u.Host + hostName, _, errIgnored := net.SplitHostPort(u.Host) + if errIgnored != nil { + hostName = u.Host // u.Host can be "host" or "host:port" } // some users only use proxy, there is no DNS resolver. it's safe to ignore the LookupIP error @@ -168,7 +167,7 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio } if setting.Migrations.MaxAttempts > 1 { - downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) + downloader = base.NewRetryDownloader(downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) } return downloader, nil } @@ -176,12 +175,12 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio // migrateRepository will download information and then upload it to Uploader, this is a simple // process for small repository. For a big repository, save all the data to disk // before upload is better -func migrateRepository(_ context.Context, doer *user_model.User, downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error { +func migrateRepository(ctx context.Context, doer *user_model.User, downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error { if messenger == nil { messenger = base.NilMessenger } - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -211,7 +210,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base if cloneURL.Scheme == "file" || cloneURL.Scheme == "" { if cloneAddrURL.Scheme != "file" && cloneAddrURL.Scheme != "" { - return fmt.Errorf("repo info has changed from external to local filesystem") + return errors.New("repo info has changed from external to local filesystem") } } @@ -220,14 +219,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base log.Trace("migrating git data from %s", repo.CloneURL) messenger("repo.migrate.migrating_git") - if err = uploader.CreateRepo(repo, opts); err != nil { + if err = uploader.CreateRepo(ctx, repo, opts); err != nil { return err } defer uploader.Close() log.Trace("migrating topics") messenger("repo.migrate.migrating_topics") - topics, err := downloader.GetTopics() + topics, err := downloader.GetTopics(ctx) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -235,7 +234,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base log.Warn("migrating topics is not supported, ignored") } if len(topics) != 0 { - if err = uploader.CreateTopics(topics...); err != nil { + if err = uploader.CreateTopics(ctx, topics...); err != nil { return err } } @@ -243,7 +242,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base if opts.Milestones { log.Trace("migrating milestones") messenger("repo.migrate.migrating_milestones") - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -256,7 +255,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base msBatchSize = len(milestones) } - if err := uploader.CreateMilestones(milestones[:msBatchSize]...); err != nil { + if err := uploader.CreateMilestones(ctx, milestones[:msBatchSize]...); err != nil { return err } milestones = milestones[msBatchSize:] @@ -266,7 +265,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base if opts.Labels { log.Trace("migrating labels") messenger("repo.migrate.migrating_labels") - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -280,7 +279,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base lbBatchSize = len(labels) } - if err := uploader.CreateLabels(labels[:lbBatchSize]...); err != nil { + if err := uploader.CreateLabels(ctx, labels[:lbBatchSize]...); err != nil { return err } labels = labels[lbBatchSize:] @@ -290,7 +289,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base if opts.Releases { log.Trace("migrating releases") messenger("repo.migrate.migrating_releases") - releases, err := downloader.GetReleases() + releases, err := downloader.GetReleases(ctx) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -304,14 +303,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base relBatchSize = len(releases) } - if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil { + if err = uploader.CreateReleases(ctx, releases[:relBatchSize]...); err != nil { return err } releases = releases[relBatchSize:] } // Once all releases (if any) are inserted, sync any remaining non-release tags - if err = uploader.SyncTags(); err != nil { + if err = uploader.SyncTags(ctx); err != nil { return err } } @@ -329,7 +328,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base issueBatchSize := uploader.MaxBatchInsertSize("issue") for i := 1; ; i++ { - issues, isEnd, err := downloader.GetIssues(i, issueBatchSize) + issues, isEnd, err := downloader.GetIssues(ctx, i, issueBatchSize) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -338,7 +337,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base break } - if err := uploader.CreateIssues(issues...); err != nil { + if err := uploader.CreateIssues(ctx, issues...); err != nil { return err } @@ -346,7 +345,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base allComments := make([]*base.Comment, 0, commentBatchSize) for _, issue := range issues { log.Trace("migrating issue %d's comments", issue.Number) - comments, _, err := downloader.GetComments(issue) + comments, _, err := downloader.GetComments(ctx, issue) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -357,7 +356,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base allComments = append(allComments, comments...) if len(allComments) >= commentBatchSize { - if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + if err = uploader.CreateComments(ctx, allComments[:commentBatchSize]...); err != nil { return err } @@ -366,7 +365,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base } if len(allComments) > 0 { - if err = uploader.CreateComments(allComments...); err != nil { + if err = uploader.CreateComments(ctx, allComments...); err != nil { return err } } @@ -383,7 +382,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base messenger("repo.migrate.migrating_pulls") prBatchSize := uploader.MaxBatchInsertSize("pullrequest") for i := 1; ; i++ { - prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize) + prs, isEnd, err := downloader.GetPullRequests(ctx, i, prBatchSize) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -392,7 +391,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base break } - if err := uploader.CreatePullRequests(prs...); err != nil { + if err := uploader.CreatePullRequests(ctx, prs...); err != nil { return err } @@ -402,7 +401,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base allComments := make([]*base.Comment, 0, commentBatchSize) for _, pr := range prs { log.Trace("migrating pull request %d's comments", pr.Number) - comments, _, err := downloader.GetComments(pr) + comments, _, err := downloader.GetComments(ctx, pr) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -413,14 +412,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base allComments = append(allComments, comments...) if len(allComments) >= commentBatchSize { - if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + if err = uploader.CreateComments(ctx, allComments[:commentBatchSize]...); err != nil { return err } allComments = allComments[commentBatchSize:] } } if len(allComments) > 0 { - if err = uploader.CreateComments(allComments...); err != nil { + if err = uploader.CreateComments(ctx, allComments...); err != nil { return err } } @@ -429,7 +428,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base // migrate reviews allReviews := make([]*base.Review, 0, reviewBatchSize) for _, pr := range prs { - reviews, err := downloader.GetReviews(pr) + reviews, err := downloader.GetReviews(ctx, pr) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -441,14 +440,14 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base allReviews = append(allReviews, reviews...) if len(allReviews) >= reviewBatchSize { - if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { + if err = uploader.CreateReviews(ctx, allReviews[:reviewBatchSize]...); err != nil { return err } allReviews = allReviews[reviewBatchSize:] } } if len(allReviews) > 0 { - if err = uploader.CreateReviews(allReviews...); err != nil { + if err = uploader.CreateReviews(ctx, allReviews...); err != nil { return err } } @@ -463,12 +462,12 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base if opts.Comments && supportAllComments { log.Trace("migrating comments") for i := 1; ; i++ { - comments, isEnd, err := downloader.GetAllComments(i, commentBatchSize) + comments, isEnd, err := downloader.GetAllComments(ctx, i, commentBatchSize) if err != nil { return err } - if err := uploader.CreateComments(comments...); err != nil { + if err := uploader.CreateComments(ctx, comments...); err != nil { return err } @@ -478,7 +477,7 @@ func migrateRepository(_ context.Context, doer *user_model.User, downloader base } } - return uploader.Finish() + return uploader.Finish(ctx) } // Init migrations service diff --git a/services/migrations/onedev.go b/services/migrations/onedev.go index e2f7b771f3..e052cba0cc 100644 --- a/services/migrations/onedev.go +++ b/services/migrations/onedev.go @@ -71,7 +71,6 @@ type onedevUser struct { // from OneDev type OneDevDownloader struct { base.NullDownloader - ctx context.Context client *http.Client baseURL *url.URL repoName string @@ -81,15 +80,9 @@ type OneDevDownloader struct { milestoneMap map[int64]string } -// SetContext set context -func (d *OneDevDownloader) SetContext(ctx context.Context) { - d.ctx = ctx -} - // NewOneDevDownloader creates a new downloader -func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader { +func NewOneDevDownloader(_ context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader { downloader := &OneDevDownloader{ - ctx: ctx, baseURL: baseURL, repoName: repoName, client: &http.Client{ @@ -121,7 +114,7 @@ func (d *OneDevDownloader) LogString() string { return fmt.Sprintf("<OneDevDownloader %s [%d]/%s>", d.baseURL, d.repoID, d.repoName) } -func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result any) error { +func (d *OneDevDownloader) callAPI(ctx context.Context, endpoint string, parameter map[string]string, result any) error { u, err := d.baseURL.Parse(endpoint) if err != nil { return err @@ -135,7 +128,7 @@ func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, u.RawQuery = query.Encode() } - req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return err } @@ -151,7 +144,7 @@ func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, } // GetRepoInfo returns repository information -func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { +func (d *OneDevDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) { info := make([]struct { ID int64 `json:"id"` Name string `json:"name"` @@ -159,6 +152,7 @@ func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { }, 0, 1) err := d.callAPI( + ctx, "/api/projects", map[string]string{ "query": `"Name" is "` + d.repoName + `"`, @@ -194,7 +188,7 @@ func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { } // GetMilestones returns milestones -func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { +func (d *OneDevDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) { rawMilestones := make([]struct { ID int64 `json:"id"` Name string `json:"name"` @@ -209,6 +203,7 @@ func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { offset := 0 for { err := d.callAPI( + ctx, endpoint, map[string]string{ "offset": strconv.Itoa(offset), @@ -243,7 +238,7 @@ func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { } // GetLabels returns labels -func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) { +func (d *OneDevDownloader) GetLabels(_ context.Context) ([]*base.Label, error) { return []*base.Label{ { Name: "Bug", @@ -277,7 +272,7 @@ type onedevIssueContext struct { } // GetIssues returns issues -func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (d *OneDevDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) { rawIssues := make([]struct { ID int64 `json:"id"` Number int64 `json:"number"` @@ -289,6 +284,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er }, 0, perPage) err := d.callAPI( + ctx, "/api/issues", map[string]string{ "query": `"Project" is "` + d.repoName + `"`, @@ -308,6 +304,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Value string `json:"value"` }, 0, 10) err := d.callAPI( + ctx, fmt.Sprintf("/api/issues/%d/fields", issue.ID), nil, &fields, @@ -329,6 +326,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Name string `json:"name"` }, 0, 10) err = d.callAPI( + ctx, fmt.Sprintf("/api/issues/%d/milestones", issue.ID), nil, &milestones, @@ -345,7 +343,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er if state == "released" { state = "closed" } - poster := d.tryGetUser(issue.SubmitterID) + poster := d.tryGetUser(ctx, issue.SubmitterID) issues = append(issues, &base.Issue{ Title: issue.Title, Number: issue.Number, @@ -370,7 +368,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er } // GetComments returns comments -func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (d *OneDevDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { context, ok := commentable.GetContext().(onedevIssueContext) if !ok { return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) @@ -391,6 +389,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co } err := d.callAPI( + ctx, endpoint, nil, &rawComments, @@ -412,6 +411,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co } err = d.callAPI( + ctx, endpoint, nil, &rawChanges, @@ -425,7 +425,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co if len(comment.Content) == 0 { continue } - poster := d.tryGetUser(comment.UserID) + poster := d.tryGetUser(ctx, comment.UserID) comments = append(comments, &base.Comment{ IssueIndex: commentable.GetLocalIndex(), Index: comment.ID, @@ -450,7 +450,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co continue } - poster := d.tryGetUser(change.UserID) + poster := d.tryGetUser(ctx, change.UserID) comments = append(comments, &base.Comment{ IssueIndex: commentable.GetLocalIndex(), PosterID: poster.ID, @@ -466,7 +466,7 @@ func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Co } // GetPullRequests returns pull requests -func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (d *OneDevDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { rawPullRequests := make([]struct { ID int64 `json:"id"` Number int64 `json:"number"` @@ -484,6 +484,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque }, 0, perPage) err := d.callAPI( + ctx, "/api/pull-requests", map[string]string{ "query": `"Target Project" is "` + d.repoName + `"`, @@ -505,6 +506,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque MergeCommitHash string `json:"mergeCommitHash"` } err := d.callAPI( + ctx, fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID), nil, &mergePreview, @@ -525,7 +527,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque mergedTime = pr.CloseInfo.Date } } - poster := d.tryGetUser(pr.SubmitterID) + poster := d.tryGetUser(ctx, pr.SubmitterID) number := pr.Number + d.maxIssueIndex pullRequests = append(pullRequests, &base.PullRequest{ @@ -562,7 +564,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests reviews -func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (d *OneDevDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { rawReviews := make([]struct { ID int64 `json:"id"` UserID int64 `json:"userId"` @@ -574,6 +576,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie }, 0, 100) err := d.callAPI( + ctx, fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignIndex()), nil, &rawReviews, @@ -596,7 +599,7 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie } } - poster := d.tryGetUser(review.UserID) + poster := d.tryGetUser(ctx, review.UserID) reviews = append(reviews, &base.Review{ IssueIndex: reviewable.GetLocalIndex(), ReviewerID: poster.ID, @@ -610,14 +613,15 @@ func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Revie } // GetTopics return repository topics -func (d *OneDevDownloader) GetTopics() ([]string, error) { +func (d *OneDevDownloader) GetTopics(_ context.Context) ([]string, error) { return []string{}, nil } -func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser { +func (d *OneDevDownloader) tryGetUser(ctx context.Context, userID int64) *onedevUser { user, ok := d.userMap[userID] if !ok { err := d.callAPI( + ctx, fmt.Sprintf("/api/users/%d", userID), nil, &user, diff --git a/services/migrations/onedev_test.go b/services/migrations/onedev_test.go index 48412fec64..a05d6cac6e 100644 --- a/services/migrations/onedev_test.go +++ b/services/migrations/onedev_test.go @@ -4,7 +4,6 @@ package migrations import ( - "context" "net/http" "net/url" "testing" @@ -22,11 +21,12 @@ func TestOneDevDownloadRepo(t *testing.T) { } u, _ := url.Parse("https://code.onedev.io") - downloader := NewOneDevDownloader(context.Background(), u, "", "", "go-gitea-test_repo") + ctx := t.Context() + downloader := NewOneDevDownloader(ctx, u, "", "", "go-gitea-test_repo") if err != nil { t.Fatalf("NewOneDevDownloader is nil: %v", err) } - repo, err := downloader.GetRepoInfo() + repo, err := downloader.GetRepoInfo(ctx) assert.NoError(t, err) assertRepositoryEqual(t, &base.Repository{ Name: "go-gitea-test_repo", @@ -36,7 +36,7 @@ func TestOneDevDownloadRepo(t *testing.T) { OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo", }, repo) - milestones, err := downloader.GetMilestones() + milestones, err := downloader.GetMilestones(ctx) assert.NoError(t, err) deadline := time.Unix(1620086400, 0) assertMilestonesEqual(t, []*base.Milestone{ @@ -51,11 +51,11 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, milestones) - labels, err := downloader.GetLabels() + labels, err := downloader.GetLabels(ctx) assert.NoError(t, err) assert.Len(t, labels, 6) - issues, isEnd, err := downloader.GetIssues(1, 2) + issues, isEnd, err := downloader.GetIssues(ctx, 1, 2) assert.NoError(t, err) assert.False(t, isEnd) assertIssuesEqual(t, []*base.Issue{ @@ -94,7 +94,7 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, issues) - comments, _, err := downloader.GetComments(&base.Issue{ + comments, _, err := downloader.GetComments(ctx, &base.Issue{ Number: 4, ForeignIndex: 398, Context: onedevIssueContext{IsPullRequest: false}, @@ -110,7 +110,7 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, comments) - prs, _, err := downloader.GetPullRequests(1, 1) + prs, _, err := downloader.GetPullRequests(ctx, 1, 1) assert.NoError(t, err) assertPullRequestsEqual(t, []*base.PullRequest{ { @@ -136,7 +136,7 @@ func TestOneDevDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(&base.PullRequest{Number: 5, ForeignIndex: 186}) + rvs, err := downloader.GetReviews(ctx, &base.PullRequest{Number: 5, ForeignIndex: 186}) assert.NoError(t, err) assertReviewsEqual(t, []*base.Review{ { diff --git a/services/migrations/restore.go b/services/migrations/restore.go index fd337b22c7..5686285935 100644 --- a/services/migrations/restore.go +++ b/services/migrations/restore.go @@ -18,7 +18,6 @@ import ( // RepositoryRestorer implements an Downloader from the local directory type RepositoryRestorer struct { base.NullDownloader - ctx context.Context baseDir string repoOwner string repoName string @@ -26,13 +25,12 @@ type RepositoryRestorer struct { } // NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder -func NewRepositoryRestorer(ctx context.Context, baseDir, owner, repoName string, validation bool) (*RepositoryRestorer, error) { +func NewRepositoryRestorer(_ context.Context, baseDir, owner, repoName string, validation bool) (*RepositoryRestorer, error) { baseDir, err := filepath.Abs(baseDir) if err != nil { return nil, err } return &RepositoryRestorer{ - ctx: ctx, baseDir: baseDir, repoOwner: owner, repoName: repoName, @@ -48,11 +46,6 @@ func (r *RepositoryRestorer) reviewDir() string { return filepath.Join(r.baseDir, "reviews") } -// SetContext set context -func (r *RepositoryRestorer) SetContext(ctx context.Context) { - r.ctx = ctx -} - func (r *RepositoryRestorer) getRepoOptions() (map[string]string, error) { p := filepath.Join(r.baseDir, "repo.yml") bs, err := os.ReadFile(p) @@ -69,7 +62,7 @@ func (r *RepositoryRestorer) getRepoOptions() (map[string]string, error) { } // GetRepoInfo returns a repository information -func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { +func (r *RepositoryRestorer) GetRepoInfo(_ context.Context) (*base.Repository, error) { opts, err := r.getRepoOptions() if err != nil { return nil, err @@ -89,7 +82,7 @@ func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) { } // GetTopics return github topics -func (r *RepositoryRestorer) GetTopics() ([]string, error) { +func (r *RepositoryRestorer) GetTopics(_ context.Context) ([]string, error) { p := filepath.Join(r.baseDir, "topic.yml") topics := struct { @@ -112,7 +105,7 @@ func (r *RepositoryRestorer) GetTopics() ([]string, error) { } // GetMilestones returns milestones -func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { +func (r *RepositoryRestorer) GetMilestones(_ context.Context) ([]*base.Milestone, error) { milestones := make([]*base.Milestone, 0, 10) p := filepath.Join(r.baseDir, "milestone.yml") err := base.Load(p, &milestones, r.validation) @@ -127,7 +120,7 @@ func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) { } // GetReleases returns releases -func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { +func (r *RepositoryRestorer) GetReleases(_ context.Context) ([]*base.Release, error) { releases := make([]*base.Release, 0, 10) p := filepath.Join(r.baseDir, "release.yml") _, err := os.Stat(p) @@ -158,7 +151,7 @@ func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) { } // GetLabels returns labels -func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { +func (r *RepositoryRestorer) GetLabels(_ context.Context) ([]*base.Label, error) { labels := make([]*base.Label, 0, 10) p := filepath.Join(r.baseDir, "label.yml") _, err := os.Stat(p) @@ -182,7 +175,7 @@ func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) { } // GetIssues returns issues according start and limit -func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { +func (r *RepositoryRestorer) GetIssues(_ context.Context, _, _ int) ([]*base.Issue, bool, error) { issues := make([]*base.Issue, 0, 10) p := filepath.Join(r.baseDir, "issue.yml") err := base.Load(p, &issues, r.validation) @@ -196,7 +189,7 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (r *RepositoryRestorer) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { +func (r *RepositoryRestorer) GetComments(_ context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { comments := make([]*base.Comment, 0, 10) p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", commentable.GetForeignIndex())) _, err := os.Stat(p) @@ -220,7 +213,7 @@ func (r *RepositoryRestorer) GetComments(commentable base.Commentable) ([]*base. } // GetPullRequests returns pull requests according page and perPage -func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { +func (r *RepositoryRestorer) GetPullRequests(_ context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { pulls := make([]*base.PullRequest, 0, 10) p := filepath.Join(r.baseDir, "pull_request.yml") _, err := os.Stat(p) @@ -248,7 +241,7 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } // GetReviews returns pull requests review -func (r *RepositoryRestorer) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) { +func (r *RepositoryRestorer) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { reviews := make([]*base.Review, 0, 10) p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", reviewable.GetForeignIndex())) _, err := os.Stat(p) |