123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702 |
- // Copyright 2020 The Gitea Authors. All rights reserved.
- // SPDX-License-Identifier: MIT
-
- package migrations
-
- import (
- "context"
- "errors"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "strings"
- "time"
-
- "code.gitea.io/gitea/modules/log"
- base "code.gitea.io/gitea/modules/migration"
- "code.gitea.io/gitea/modules/structs"
-
- gitea_sdk "code.gitea.io/sdk/gitea"
- )
-
- var (
- _ base.Downloader = &GiteaDownloader{}
- _ base.DownloaderFactory = &GiteaDownloaderFactory{}
- )
-
- func init() {
- RegisterDownloaderFactory(&GiteaDownloaderFactory{})
- }
-
- // GiteaDownloaderFactory defines a gitea downloader factory
- type GiteaDownloaderFactory struct{}
-
- // New returns a Downloader related to this factory according MigrateOptions
- func (f *GiteaDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
- u, err := url.Parse(opts.CloneAddr)
- if err != nil {
- return nil, err
- }
-
- baseURL := u.Scheme + "://" + u.Host
- repoNameSpace := strings.TrimPrefix(u.Path, "/")
- repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
-
- path := strings.Split(repoNameSpace, "/")
- if len(path) < 2 {
- return nil, fmt.Errorf("invalid path: %s", repoNameSpace)
- }
-
- repoPath := strings.Join(path[len(path)-2:], "/")
- if len(path) > 2 {
- subPath := strings.Join(path[:len(path)-2], "/")
- baseURL += "/" + subPath
- }
-
- log.Trace("Create gitea downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
-
- return NewGiteaDownloader(ctx, baseURL, repoPath, opts.AuthUsername, opts.AuthPassword, opts.AuthToken)
- }
-
- // GitServiceType returns the type of git service
- func (f *GiteaDownloaderFactory) GitServiceType() structs.GitServiceType {
- return structs.GiteaService
- }
-
- // GiteaDownloader implements a Downloader interface to get repository information's
- type GiteaDownloader struct {
- base.NullDownloader
- ctx context.Context
- client *gitea_sdk.Client
- baseURL string
- repoOwner string
- repoName string
- pagination bool
- maxPerPage int
- }
-
- // NewGiteaDownloader creates a gitea Downloader via gitea API
- //
- // Use either a username/password or personal token. token is preferred
- // Note: Public access only allows very basic access
- func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GiteaDownloader, error) {
- giteaClient, err := gitea_sdk.NewClient(
- baseURL,
- gitea_sdk.SetToken(token),
- gitea_sdk.SetBasicAuth(username, password),
- gitea_sdk.SetContext(ctx),
- gitea_sdk.SetHTTPClient(NewMigrationHTTPClient()),
- )
- if err != nil {
- log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
- return nil, err
- }
-
- path := strings.Split(repoPath, "/")
-
- paginationSupport := true
- if err = giteaClient.CheckServerVersionConstraint(">=1.12"); err != nil {
- paginationSupport = false
- }
-
- // set small maxPerPage since we can only guess
- // (default would be 50 but this can differ)
- maxPerPage := 10
- // gitea instances >=1.13 can tell us what maximum they have
- apiConf, _, err := giteaClient.GetGlobalAPISettings()
- if err != nil {
- log.Info("Unable to get global API settings. Ignoring these.")
- log.Debug("giteaClient.GetGlobalAPISettings. Error: %v", err)
- }
- if apiConf != nil {
- maxPerPage = apiConf.MaxResponseItems
- }
-
- return &GiteaDownloader{
- ctx: ctx,
- client: giteaClient,
- baseURL: baseURL,
- repoOwner: path[0],
- repoName: path[1],
- pagination: paginationSupport,
- maxPerPage: maxPerPage,
- }, nil
- }
-
- // SetContext set context
- func (g *GiteaDownloader) SetContext(ctx context.Context) {
- g.ctx = ctx
- }
-
- // String implements Stringer
- func (g *GiteaDownloader) String() string {
- return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
- }
-
- func (g *GiteaDownloader) LogString() string {
- if g == nil {
- return "<GiteaDownloader nil>"
- }
- return fmt.Sprintf("<GiteaDownloader %s %s/%s>", g.baseURL, g.repoOwner, g.repoName)
- }
-
- // GetRepoInfo returns a repository information
- func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) {
- if g == nil {
- return nil, errors.New("error: GiteaDownloader is nil")
- }
-
- repo, _, err := g.client.GetRepo(g.repoOwner, g.repoName)
- if err != nil {
- return nil, err
- }
-
- return &base.Repository{
- Name: repo.Name,
- Owner: repo.Owner.UserName,
- IsPrivate: repo.Private,
- Description: repo.Description,
- CloneURL: repo.CloneURL,
- OriginalURL: repo.HTMLURL,
- DefaultBranch: repo.DefaultBranch,
- }, nil
- }
-
- // GetTopics return gitea topics
- func (g *GiteaDownloader) GetTopics() ([]string, error) {
- topics, _, err := g.client.ListRepoTopics(g.repoOwner, g.repoName, gitea_sdk.ListRepoTopicsOptions{})
- return topics, err
- }
-
- // GetMilestones returns milestones
- func (g *GiteaDownloader) GetMilestones() ([]*base.Milestone, error) {
- milestones := make([]*base.Milestone, 0, g.maxPerPage)
-
- for i := 1; ; i++ {
- // make sure gitea can shutdown gracefully
- select {
- case <-g.ctx.Done():
- return nil, nil
- default:
- }
-
- ms, _, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName, gitea_sdk.ListMilestoneOption{
- ListOptions: gitea_sdk.ListOptions{
- PageSize: g.maxPerPage,
- Page: i,
- },
- State: gitea_sdk.StateAll,
- })
- if err != nil {
- return nil, err
- }
-
- for i := range ms {
- // old gitea instances dont have this information
- createdAT := time.Time{}
- var updatedAT *time.Time
- if ms[i].Closed != nil {
- createdAT = *ms[i].Closed
- updatedAT = ms[i].Closed
- }
-
- // new gitea instances (>=1.13) do
- if !ms[i].Created.IsZero() {
- createdAT = ms[i].Created
- }
- if ms[i].Updated != nil && !ms[i].Updated.IsZero() {
- updatedAT = ms[i].Updated
- }
-
- milestones = append(milestones, &base.Milestone{
- Title: ms[i].Title,
- Description: ms[i].Description,
- Deadline: ms[i].Deadline,
- Created: createdAT,
- Updated: updatedAT,
- Closed: ms[i].Closed,
- State: string(ms[i].State),
- })
- }
- if !g.pagination || len(ms) < g.maxPerPage {
- break
- }
- }
- return milestones, nil
- }
-
- func (g *GiteaDownloader) convertGiteaLabel(label *gitea_sdk.Label) *base.Label {
- return &base.Label{
- Name: label.Name,
- Color: label.Color,
- Description: label.Description,
- }
- }
-
- // GetLabels returns labels
- func (g *GiteaDownloader) GetLabels() ([]*base.Label, error) {
- labels := make([]*base.Label, 0, g.maxPerPage)
-
- for i := 1; ; i++ {
- // make sure gitea can shutdown gracefully
- select {
- case <-g.ctx.Done():
- return nil, nil
- default:
- }
-
- ls, _, err := g.client.ListRepoLabels(g.repoOwner, g.repoName, gitea_sdk.ListLabelsOptions{ListOptions: gitea_sdk.ListOptions{
- PageSize: g.maxPerPage,
- Page: i,
- }})
- if err != nil {
- return nil, err
- }
-
- for i := range ls {
- labels = append(labels, g.convertGiteaLabel(ls[i]))
- }
- if !g.pagination || len(ls) < g.maxPerPage {
- break
- }
- }
- return labels, nil
- }
-
- func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Release {
- r := &base.Release{
- TagName: rel.TagName,
- TargetCommitish: rel.Target,
- Name: rel.Title,
- Body: rel.Note,
- Draft: rel.IsDraft,
- Prerelease: rel.IsPrerelease,
- PublisherID: rel.Publisher.ID,
- PublisherName: rel.Publisher.UserName,
- PublisherEmail: rel.Publisher.Email,
- Published: rel.PublishedAt,
- Created: rel.CreatedAt,
- }
-
- httpClient := NewMigrationHTTPClient()
-
- for _, asset := range rel.Attachments {
- size := int(asset.Size)
- dlCount := int(asset.DownloadCount)
- r.Assets = append(r.Assets, &base.ReleaseAsset{
- ID: asset.ID,
- Name: asset.Name,
- Size: &size,
- DownloadCount: &dlCount,
- Created: asset.Created,
- DownloadURL: &asset.DownloadURL,
- DownloadFunc: func() (io.ReadCloser, error) {
- asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, rel.ID, asset.ID)
- if err != nil {
- return nil, err
- }
-
- if !hasBaseURL(asset.DownloadURL, g.baseURL) {
- WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", asset.ID, g, asset.DownloadURL)
- return io.NopCloser(strings.NewReader(asset.DownloadURL)), nil
- }
-
- // FIXME: for a private download?
- req, err := http.NewRequest("GET", asset.DownloadURL, nil)
- if err != nil {
- return nil, err
- }
- resp, err := httpClient.Do(req)
- if err != nil {
- return nil, err
- }
-
- // resp.Body is closed by the uploader
- return resp.Body, nil
- },
- })
- }
- return r
- }
-
- // GetReleases returns releases
- func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) {
- releases := make([]*base.Release, 0, g.maxPerPage)
-
- for i := 1; ; i++ {
- // make sure gitea can shutdown gracefully
- select {
- case <-g.ctx.Done():
- return nil, nil
- default:
- }
-
- rl, _, err := g.client.ListReleases(g.repoOwner, g.repoName, gitea_sdk.ListReleasesOptions{ListOptions: gitea_sdk.ListOptions{
- PageSize: g.maxPerPage,
- Page: i,
- }})
- if err != nil {
- return nil, err
- }
-
- for i := range rl {
- releases = append(releases, g.convertGiteaRelease(rl[i]))
- }
- if !g.pagination || len(rl) < g.maxPerPage {
- break
- }
- }
- return releases, nil
- }
-
- func (g *GiteaDownloader) getIssueReactions(index int64) ([]*base.Reaction, error) {
- var reactions []*base.Reaction
- if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
- log.Info("GiteaDownloader: instance to old, skip getIssueReactions")
- return reactions, nil
- }
- rl, _, err := g.client.GetIssueReactions(g.repoOwner, g.repoName, index)
- if err != nil {
- return nil, err
- }
-
- for _, reaction := range rl {
- reactions = append(reactions, &base.Reaction{
- UserID: reaction.User.ID,
- UserName: reaction.User.UserName,
- Content: reaction.Reaction,
- })
- }
- return reactions, nil
- }
-
- func (g *GiteaDownloader) getCommentReactions(commentID int64) ([]*base.Reaction, error) {
- var reactions []*base.Reaction
- if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
- log.Info("GiteaDownloader: instance to old, skip getCommentReactions")
- return reactions, nil
- }
- rl, _, err := g.client.GetIssueCommentReactions(g.repoOwner, g.repoName, commentID)
- if err != nil {
- return nil, err
- }
-
- for i := range rl {
- reactions = append(reactions, &base.Reaction{
- UserID: rl[i].User.ID,
- UserName: rl[i].User.UserName,
- Content: rl[i].Reaction,
- })
- }
- return reactions, nil
- }
-
- // GetIssues returns issues according start and limit
- func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
- allIssues := make([]*base.Issue, 0, perPage)
-
- issues, _, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gitea_sdk.ListIssueOption{
- ListOptions: gitea_sdk.ListOptions{Page: page, PageSize: perPage},
- State: gitea_sdk.StateAll,
- Type: gitea_sdk.IssueTypeIssue,
- })
- if err != nil {
- return nil, false, fmt.Errorf("error while listing issues: %w", err)
- }
- for _, issue := range issues {
-
- labels := make([]*base.Label, 0, len(issue.Labels))
- for i := range issue.Labels {
- labels = append(labels, g.convertGiteaLabel(issue.Labels[i]))
- }
-
- var milestone string
- if issue.Milestone != nil {
- milestone = issue.Milestone.Title
- }
-
- reactions, err := g.getIssueReactions(issue.Index)
- if err != nil {
- WarnAndNotice("Unable to load reactions during migrating issue #%d in %s. Error: %v", issue.Index, g, err)
- }
-
- var assignees []string
- for i := range issue.Assignees {
- assignees = append(assignees, issue.Assignees[i].UserName)
- }
-
- allIssues = append(allIssues, &base.Issue{
- Title: issue.Title,
- Number: issue.Index,
- PosterID: issue.Poster.ID,
- PosterName: issue.Poster.UserName,
- PosterEmail: issue.Poster.Email,
- Content: issue.Body,
- Milestone: milestone,
- State: string(issue.State),
- Created: issue.Created,
- Updated: issue.Updated,
- Closed: issue.Closed,
- Reactions: reactions,
- Labels: labels,
- Assignees: assignees,
- IsLocked: issue.IsLocked,
- ForeignIndex: issue.Index,
- })
- }
-
- isEnd := len(issues) < perPage
- if !g.pagination {
- isEnd = len(issues) == 0
- }
- return allIssues, isEnd, nil
- }
-
- // GetComments returns comments according issueNumber
- func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) {
- allComments := make([]*base.Comment, 0, g.maxPerPage)
-
- for i := 1; ; i++ {
- // make sure gitea can shutdown gracefully
- select {
- case <-g.ctx.Done():
- return nil, false, nil
- default:
- }
-
- comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, commentable.GetForeignIndex(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{
- PageSize: g.maxPerPage,
- Page: i,
- }})
- if err != nil {
- return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %w", commentable.GetForeignIndex(), err)
- }
-
- for _, comment := range comments {
- reactions, err := g.getCommentReactions(comment.ID)
- if err != nil {
- WarnAndNotice("Unable to load comment reactions during migrating issue #%d for comment %d in %s. Error: %v", commentable.GetForeignIndex(), comment.ID, g, err)
- }
-
- allComments = append(allComments, &base.Comment{
- IssueIndex: commentable.GetLocalIndex(),
- Index: comment.ID,
- PosterID: comment.Poster.ID,
- PosterName: comment.Poster.UserName,
- PosterEmail: comment.Poster.Email,
- Content: comment.Body,
- Created: comment.Created,
- Updated: comment.Updated,
- Reactions: reactions,
- })
- }
-
- if !g.pagination || len(comments) < g.maxPerPage {
- break
- }
- }
- return allComments, true, nil
- }
-
- // GetPullRequests returns pull requests according page and perPage
- func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
- allPRs := make([]*base.PullRequest, 0, perPage)
-
- prs, _, err := g.client.ListRepoPullRequests(g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
- ListOptions: gitea_sdk.ListOptions{
- Page: page,
- PageSize: perPage,
- },
- State: gitea_sdk.StateAll,
- })
- if err != nil {
- return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %w", page, perPage, err)
- }
- for _, pr := range prs {
- var milestone string
- if pr.Milestone != nil {
- milestone = pr.Milestone.Title
- }
-
- labels := make([]*base.Label, 0, len(pr.Labels))
- for i := range pr.Labels {
- labels = append(labels, g.convertGiteaLabel(pr.Labels[i]))
- }
-
- var (
- headUserName string
- headRepoName string
- headCloneURL string
- headRef string
- headSHA string
- )
- if pr.Head != nil {
- if pr.Head.Repository != nil {
- headUserName = pr.Head.Repository.Owner.UserName
- headRepoName = pr.Head.Repository.Name
- headCloneURL = pr.Head.Repository.CloneURL
- }
- headSHA = pr.Head.Sha
- headRef = pr.Head.Ref
- }
-
- var mergeCommitSHA string
- if pr.MergedCommitID != nil {
- mergeCommitSHA = *pr.MergedCommitID
- }
-
- reactions, err := g.getIssueReactions(pr.Index)
- if err != nil {
- WarnAndNotice("Unable to load reactions during migrating pull #%d in %s. Error: %v", pr.Index, g, err)
- }
-
- var assignees []string
- for i := range pr.Assignees {
- assignees = append(assignees, pr.Assignees[i].UserName)
- }
-
- createdAt := time.Time{}
- if pr.Created != nil {
- createdAt = *pr.Created
- }
- updatedAt := time.Time{}
- if pr.Created != nil {
- updatedAt = *pr.Updated
- }
-
- closedAt := pr.Closed
- if pr.Merged != nil && closedAt == nil {
- closedAt = pr.Merged
- }
-
- allPRs = append(allPRs, &base.PullRequest{
- Title: pr.Title,
- Number: pr.Index,
- PosterID: pr.Poster.ID,
- PosterName: pr.Poster.UserName,
- PosterEmail: pr.Poster.Email,
- Content: pr.Body,
- State: string(pr.State),
- Created: createdAt,
- Updated: updatedAt,
- Closed: closedAt,
- Labels: labels,
- Milestone: milestone,
- Reactions: reactions,
- Assignees: assignees,
- Merged: pr.HasMerged,
- MergedTime: pr.Merged,
- MergeCommitSHA: mergeCommitSHA,
- IsLocked: pr.IsLocked,
- PatchURL: pr.PatchURL,
- Head: base.PullRequestBranch{
- Ref: headRef,
- SHA: headSHA,
- RepoName: headRepoName,
- OwnerName: headUserName,
- CloneURL: headCloneURL,
- },
- Base: base.PullRequestBranch{
- Ref: pr.Base.Ref,
- SHA: pr.Base.Sha,
- RepoName: g.repoName,
- OwnerName: g.repoOwner,
- },
- ForeignIndex: pr.Index,
- })
- // SECURITY: Ensure that the PR is safe
- _ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g)
- }
-
- isEnd := len(prs) < perPage
- if !g.pagination {
- isEnd = len(prs) == 0
- }
- return allPRs, isEnd, nil
- }
-
- // GetReviews returns pull requests review
- func (g *GiteaDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) {
- if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil {
- log.Info("GiteaDownloader: instance to old, skip GetReviews")
- return nil, nil
- }
-
- allReviews := make([]*base.Review, 0, g.maxPerPage)
-
- for i := 1; ; i++ {
- // make sure gitea can shutdown gracefully
- select {
- case <-g.ctx.Done():
- return nil, nil
- default:
- }
-
- prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, reviewable.GetForeignIndex(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{
- Page: i,
- PageSize: g.maxPerPage,
- }})
- if err != nil {
- return nil, err
- }
-
- for _, pr := range prl {
- if pr.Reviewer == nil {
- // Presumably this is a team review which we cannot migrate at present but we have to skip this review as otherwise the review will be mapped on to an incorrect user.
- // TODO: handle team reviews
- continue
- }
-
- rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID)
- if err != nil {
- return nil, err
- }
- var reviewComments []*base.ReviewComment
- for i := range rcl {
- line := int(rcl[i].LineNum)
- if rcl[i].OldLineNum > 0 {
- line = int(rcl[i].OldLineNum) * -1
- }
-
- reviewComments = append(reviewComments, &base.ReviewComment{
- ID: rcl[i].ID,
- Content: rcl[i].Body,
- TreePath: rcl[i].Path,
- DiffHunk: rcl[i].DiffHunk,
- Line: line,
- CommitID: rcl[i].CommitID,
- PosterID: rcl[i].Reviewer.ID,
- CreatedAt: rcl[i].Created,
- UpdatedAt: rcl[i].Updated,
- })
- }
-
- review := &base.Review{
- ID: pr.ID,
- IssueIndex: reviewable.GetLocalIndex(),
- ReviewerID: pr.Reviewer.ID,
- ReviewerName: pr.Reviewer.UserName,
- Official: pr.Official,
- CommitID: pr.CommitID,
- Content: pr.Body,
- CreatedAt: pr.Submitted,
- State: string(pr.State),
- Comments: reviewComments,
- }
-
- allReviews = append(allReviews, review)
- }
-
- if len(prl) < g.maxPerPage {
- break
- }
- }
- return allReviews, nil
- }
|