123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672 |
- // Copyright 2019 The Gitea Authors. All rights reserved.
- // Use of this source code is governed by a MIT-style
- // license that can be found in the LICENSE file.
-
- package migrations
-
- import (
- "context"
- "crypto/tls"
- "errors"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "path"
- "strings"
- "time"
-
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/proxy"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/structs"
-
- "github.com/xanzy/go-gitlab"
- )
-
- var (
- _ base.Downloader = &GitlabDownloader{}
- _ base.DownloaderFactory = &GitlabDownloaderFactory{}
- )
-
- func init() {
- RegisterDownloaderFactory(&GitlabDownloaderFactory{})
- }
-
- // GitlabDownloaderFactory defines a gitlab downloader factory
- type GitlabDownloaderFactory struct {
- }
-
- // New returns a Downloader related to this factory according MigrateOptions
- func (f *GitlabDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
- u, err := url.Parse(opts.CloneAddr)
- if err != nil {
- return nil, err
- }
-
- baseURL := u.Scheme + "://" + u.Host
- repoNameSpace := strings.TrimPrefix(u.Path, "/")
- repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
-
- log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
-
- return NewGitlabDownloader(ctx, baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken)
- }
-
- // GitServiceType returns the type of git service
- func (f *GitlabDownloaderFactory) GitServiceType() structs.GitServiceType {
- return structs.GitlabService
- }
-
- // GitlabDownloader implements a Downloader interface to get repository information
- // from gitlab via go-gitlab
- // - issueCount is incremented in GetIssues() to ensure PR and Issue numbers do not overlap,
- // because Gitlab has individual Issue and Pull Request numbers.
- type GitlabDownloader struct {
- base.NullDownloader
- ctx context.Context
- client *gitlab.Client
- repoID int
- repoName string
- issueCount int64
- maxPerPage int
- }
-
- // NewGitlabDownloader creates a gitlab Downloader via gitlab API
- // Use either a username/password, personal token entered into the username field, or anonymous/public access
- // Note: Public access only allows very basic access
- func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
- gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
- Proxy: proxy.Proxy(),
- },
- }))
- // Only use basic auth if token is blank and password is NOT
- // Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
- if token == "" && password != "" {
- gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
- }
-
- if err != nil {
- log.Trace("Error logging into gitlab: %v", err)
- return nil, err
- }
-
- // split namespace and subdirectory
- pathParts := strings.Split(strings.Trim(repoPath, "/"), "/")
- var resp *gitlab.Response
- u, _ := url.Parse(baseURL)
- for len(pathParts) >= 2 {
- _, resp, err = gitlabClient.Version.GetVersion()
- if err == nil || resp != nil && resp.StatusCode == 401 {
- err = nil // if no authentication given, this still should work
- break
- }
-
- u.Path = path.Join(u.Path, pathParts[0])
- baseURL = u.String()
- pathParts = pathParts[1:]
- _ = gitlab.WithBaseURL(baseURL)(gitlabClient)
- repoPath = strings.Join(pathParts, "/")
- }
- if err != nil {
- log.Trace("Error could not get gitlab version: %v", err)
- return nil, err
- }
-
- log.Trace("gitlab downloader: use BaseURL: '%s' and RepoPath: '%s'", baseURL, repoPath)
-
- // Grab and store project/repo ID here, due to issues using the URL escaped path
- gr, _, err := gitlabClient.Projects.GetProject(repoPath, nil, nil, gitlab.WithContext(ctx))
- if err != nil {
- log.Trace("Error retrieving project: %v", err)
- return nil, err
- }
-
- if gr == nil {
- log.Trace("Error getting project, project is nil")
- return nil, errors.New("Error getting project, project is nil")
- }
-
- return &GitlabDownloader{
- ctx: ctx,
- client: gitlabClient,
- repoID: gr.ID,
- repoName: gr.Name,
- maxPerPage: 100,
- }, nil
- }
-
- // SetContext set context
- func (g *GitlabDownloader) SetContext(ctx context.Context) {
- g.ctx = ctx
- }
-
- // GetRepoInfo returns a repository information
- func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) {
- gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
-
- var private bool
- switch gr.Visibility {
- case gitlab.InternalVisibility:
- private = true
- case gitlab.PrivateVisibility:
- private = true
- }
-
- var owner string
- if gr.Owner == nil {
- log.Trace("gr.Owner is nil, trying to get owner from Namespace")
- if gr.Namespace != nil && gr.Namespace.Kind == "user" {
- owner = gr.Namespace.Path
- }
- } else {
- owner = gr.Owner.Username
- }
-
- // convert gitlab repo to stand Repo
- return &base.Repository{
- Owner: owner,
- Name: gr.Name,
- IsPrivate: private,
- Description: gr.Description,
- OriginalURL: gr.WebURL,
- CloneURL: gr.HTTPURLToRepo,
- DefaultBranch: gr.DefaultBranch,
- }, nil
- }
-
- // GetTopics return gitlab topics
- func (g *GitlabDownloader) GetTopics() ([]string, error) {
- gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
- return gr.TagList, err
- }
-
- // GetMilestones returns milestones
- func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
- var perPage = g.maxPerPage
- var state = "all"
- var milestones = make([]*base.Milestone, 0, perPage)
- for i := 1; ; i++ {
- ms, _, err := g.client.Milestones.ListMilestones(g.repoID, &gitlab.ListMilestonesOptions{
- State: &state,
- ListOptions: gitlab.ListOptions{
- Page: i,
- PerPage: perPage,
- }}, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
-
- for _, m := range ms {
- var desc string
- if m.Description != "" {
- desc = m.Description
- }
- var state = "open"
- var closedAt *time.Time
- if m.State != "" {
- state = m.State
- if state == "closed" {
- closedAt = m.UpdatedAt
- }
- }
-
- var deadline *time.Time
- if m.DueDate != nil {
- deadlineParsed, err := time.Parse("2006-01-02", m.DueDate.String())
- if err != nil {
- log.Trace("Error parsing Milestone DueDate time")
- deadline = nil
- } else {
- deadline = &deadlineParsed
- }
- }
-
- milestones = append(milestones, &base.Milestone{
- Title: m.Title,
- Description: desc,
- Deadline: deadline,
- State: state,
- Created: *m.CreatedAt,
- Updated: m.UpdatedAt,
- Closed: closedAt,
- })
- }
- if len(ms) < perPage {
- break
- }
- }
- return milestones, nil
- }
-
- func (g *GitlabDownloader) normalizeColor(val string) string {
- val = strings.TrimLeft(val, "#")
- val = strings.ToLower(val)
- if len(val) == 3 {
- c := []rune(val)
- val = fmt.Sprintf("%c%c%c%c%c%c", c[0], c[0], c[1], c[1], c[2], c[2])
- }
- if len(val) != 6 {
- return ""
- }
- return val
- }
-
- // GetLabels returns labels
- func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
- var perPage = g.maxPerPage
- var labels = make([]*base.Label, 0, perPage)
- for i := 1; ; i++ {
- ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{
- Page: i,
- PerPage: perPage,
- }}, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
- for _, label := range ls {
- baseLabel := &base.Label{
- Name: label.Name,
- Color: g.normalizeColor(label.Color),
- Description: label.Description,
- }
- labels = append(labels, baseLabel)
- }
- if len(ls) < perPage {
- break
- }
- }
- return labels, nil
- }
-
- func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release {
- var zero int
- r := &base.Release{
- TagName: rel.TagName,
- TargetCommitish: rel.Commit.ID,
- Name: rel.Name,
- Body: rel.Description,
- Created: *rel.CreatedAt,
- PublisherID: int64(rel.Author.ID),
- PublisherName: rel.Author.Username,
- }
-
- httpClient := &http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
- Proxy: proxy.Proxy(),
- },
- }
-
- for k, asset := range rel.Assets.Links {
- r.Assets = append(r.Assets, &base.ReleaseAsset{
- ID: int64(asset.ID),
- Name: asset.Name,
- ContentType: &rel.Assets.Sources[k].Format,
- Size: &zero,
- DownloadCount: &zero,
- DownloadFunc: func() (io.ReadCloser, error) {
- link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, asset.ID, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
-
- req, err := http.NewRequest("GET", link.URL, nil)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(g.ctx)
- resp, err := httpClient.Do(req)
- if err != nil {
- return nil, err
- }
-
- // resp.Body is closed by the uploader
- return resp.Body, nil
- },
- })
- }
- return r
- }
-
- // GetReleases returns releases
- func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
- var perPage = g.maxPerPage
- var releases = make([]*base.Release, 0, perPage)
- for i := 1; ; i++ {
- ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{
- Page: i,
- PerPage: perPage,
- }, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
-
- for _, release := range ls {
- releases = append(releases, g.convertGitlabRelease(release))
- }
- if len(ls) < perPage {
- break
- }
- }
- return releases, nil
- }
-
- type gitlabIssueContext struct {
- foreignID int64
- localID int64
- IsMergeRequest bool
- }
-
- func (c gitlabIssueContext) LocalID() int64 {
- return c.localID
- }
-
- func (c gitlabIssueContext) ForeignID() int64 {
- return c.foreignID
- }
-
- // GetIssues returns issues according start and limit
- // Note: issue label description and colors are not supported by the go-gitlab library at this time
- func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
- state := "all"
- sort := "asc"
-
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
-
- opt := &gitlab.ListProjectIssuesOptions{
- State: &state,
- Sort: &sort,
- ListOptions: gitlab.ListOptions{
- PerPage: perPage,
- Page: page,
- },
- }
-
- var allIssues = make([]*base.Issue, 0, perPage)
-
- issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, false, fmt.Errorf("error while listing issues: %v", err)
- }
- for _, issue := range issues {
-
- var labels = make([]*base.Label, 0, len(issue.Labels))
- for _, l := range issue.Labels {
- labels = append(labels, &base.Label{
- Name: l,
- })
- }
-
- var milestone string
- if issue.Milestone != nil {
- milestone = issue.Milestone.Title
- }
-
- var reactions []*base.Reaction
- var awardPage = 1
- for {
- awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, false, fmt.Errorf("error while listing issue awards: %v", err)
- }
- if len(awards) < perPage {
- break
- }
- for i := range awards {
- reactions = append(reactions, g.awardToReaction(awards[i]))
- }
- awardPage++
- }
-
- allIssues = append(allIssues, &base.Issue{
- Title: issue.Title,
- Number: int64(issue.IID),
- PosterID: int64(issue.Author.ID),
- PosterName: issue.Author.Username,
- Content: issue.Description,
- Milestone: milestone,
- State: issue.State,
- Created: *issue.CreatedAt,
- Labels: labels,
- Reactions: reactions,
- Closed: issue.ClosedAt,
- IsLocked: issue.DiscussionLocked,
- Updated: *issue.UpdatedAt,
- Context: gitlabIssueContext{
- foreignID: int64(issue.IID),
- localID: int64(issue.IID),
- IsMergeRequest: false,
- },
- })
-
- // increment issueCount, to be used in GetPullRequests()
- g.issueCount++
- }
-
- return allIssues, len(issues) < perPage, nil
- }
-
- // GetComments returns comments according issueNumber
- // TODO: figure out how to transfer comment reactions
- func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
- context, ok := opts.Context.(gitlabIssueContext)
- if !ok {
- return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context)
- }
-
- var allComments = make([]*base.Comment, 0, g.maxPerPage)
-
- var page = 1
-
- for {
- var comments []*gitlab.Discussion
- var resp *gitlab.Response
- var err error
- if !context.IsMergeRequest {
- comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListIssueDiscussionsOptions{
- Page: page,
- PerPage: g.maxPerPage,
- }, nil, gitlab.WithContext(g.ctx))
- } else {
- comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListMergeRequestDiscussionsOptions{
- Page: page,
- PerPage: g.maxPerPage,
- }, nil, gitlab.WithContext(g.ctx))
- }
-
- if err != nil {
- return nil, false, fmt.Errorf("error while listing comments: %v %v", g.repoID, err)
- }
- for _, comment := range comments {
- // Flatten comment threads
- if !comment.IndividualNote {
- for _, note := range comment.Notes {
- allComments = append(allComments, &base.Comment{
- IssueIndex: context.LocalID(),
- PosterID: int64(note.Author.ID),
- PosterName: note.Author.Username,
- PosterEmail: note.Author.Email,
- Content: note.Body,
- Created: *note.CreatedAt,
- })
- }
- } else {
- c := comment.Notes[0]
- allComments = append(allComments, &base.Comment{
- IssueIndex: context.LocalID(),
- PosterID: int64(c.Author.ID),
- PosterName: c.Author.Username,
- PosterEmail: c.Author.Email,
- Content: c.Body,
- Created: *c.CreatedAt,
- })
- }
-
- }
- if resp.NextPage == 0 {
- break
- }
- page = resp.NextPage
- }
- return allComments, true, nil
- }
-
- // GetPullRequests returns pull requests according page and perPage
- func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
-
- opt := &gitlab.ListProjectMergeRequestsOptions{
- ListOptions: gitlab.ListOptions{
- PerPage: perPage,
- Page: page,
- },
- }
-
- var allPRs = make([]*base.PullRequest, 0, perPage)
-
- prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, false, fmt.Errorf("error while listing merge requests: %v", err)
- }
- for _, pr := range prs {
-
- var labels = make([]*base.Label, 0, len(pr.Labels))
- for _, l := range pr.Labels {
- labels = append(labels, &base.Label{
- Name: l,
- })
- }
-
- var merged bool
- if pr.State == "merged" {
- merged = true
- pr.State = "closed"
- }
-
- var mergeTime = pr.MergedAt
- if merged && pr.MergedAt == nil {
- mergeTime = pr.UpdatedAt
- }
-
- var closeTime = pr.ClosedAt
- if merged && pr.ClosedAt == nil {
- closeTime = pr.UpdatedAt
- }
-
- var locked bool
- if pr.State == "locked" {
- locked = true
- }
-
- var milestone string
- if pr.Milestone != nil {
- milestone = pr.Milestone.Title
- }
-
- var reactions []*base.Reaction
- var awardPage = 1
- for {
- awards, _, err := g.client.AwardEmoji.ListMergeRequestAwardEmoji(g.repoID, pr.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, false, fmt.Errorf("error while listing merge requests awards: %v", err)
- }
- if len(awards) < perPage {
- break
- }
- for i := range awards {
- reactions = append(reactions, g.awardToReaction(awards[i]))
- }
- awardPage++
- }
-
- // Add the PR ID to the Issue Count because PR and Issues share ID space in Gitea
- newPRNumber := g.issueCount + int64(pr.IID)
-
- allPRs = append(allPRs, &base.PullRequest{
- Title: pr.Title,
- Number: newPRNumber,
- PosterName: pr.Author.Username,
- PosterID: int64(pr.Author.ID),
- Content: pr.Description,
- Milestone: milestone,
- State: pr.State,
- Created: *pr.CreatedAt,
- Closed: closeTime,
- Labels: labels,
- Merged: merged,
- MergeCommitSHA: pr.MergeCommitSHA,
- MergedTime: mergeTime,
- IsLocked: locked,
- Reactions: reactions,
- Head: base.PullRequestBranch{
- Ref: pr.SourceBranch,
- SHA: pr.SHA,
- RepoName: g.repoName,
- OwnerName: pr.Author.Username,
- CloneURL: pr.WebURL,
- },
- Base: base.PullRequestBranch{
- Ref: pr.TargetBranch,
- SHA: pr.DiffRefs.BaseSha,
- RepoName: g.repoName,
- OwnerName: pr.Author.Username,
- },
- PatchURL: pr.WebURL + ".patch",
- Context: gitlabIssueContext{
- foreignID: int64(pr.IID),
- localID: newPRNumber,
- IsMergeRequest: true,
- },
- })
- }
-
- return allPRs, len(prs) < perPage, nil
- }
-
- // GetReviews returns pull requests review
- func (g *GitlabDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
- approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(context.ForeignID()), gitlab.WithContext(g.ctx))
- if err != nil {
- if resp != nil && resp.StatusCode == 404 {
- log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error()))
- return []*base.Review{}, nil
- }
- return nil, err
- }
-
- var reviews = make([]*base.Review, 0, len(approvals.ApprovedBy))
- for _, user := range approvals.ApprovedBy {
- reviews = append(reviews, &base.Review{
- IssueIndex: context.LocalID(),
- ReviewerID: int64(user.User.ID),
- ReviewerName: user.User.Username,
- CreatedAt: *approvals.UpdatedAt,
- // All we get are approvals
- State: base.ReviewStateApproved,
- })
- }
-
- return reviews, nil
- }
-
- func (g *GitlabDownloader) awardToReaction(award *gitlab.AwardEmoji) *base.Reaction {
- return &base.Reaction{
- UserID: int64(award.User.ID),
- UserName: award.User.Username,
- Content: award.Name,
- }
- }
|