123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636 |
- // Copyright 2021 The Gitea Authors. All rights reserved.
- // Use of this source code is governed by a MIT-style
- // license that can be found in the LICENSE file.
-
- package migrations
-
- import (
- "context"
- "encoding/xml"
- "fmt"
- "net/http"
- "net/url"
- "strconv"
- "strings"
- "time"
-
- "code.gitea.io/gitea/modules/log"
- base "code.gitea.io/gitea/modules/migration"
- "code.gitea.io/gitea/modules/proxy"
- "code.gitea.io/gitea/modules/structs"
- )
-
- var (
- _ base.Downloader = &CodebaseDownloader{}
- _ base.DownloaderFactory = &CodebaseDownloaderFactory{}
- )
-
- func init() {
- RegisterDownloaderFactory(&CodebaseDownloaderFactory{})
- }
-
- // CodebaseDownloaderFactory defines a downloader factory
- type CodebaseDownloaderFactory struct{}
-
- // New returns a downloader related to this factory according MigrateOptions
- func (f *CodebaseDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
- u, err := url.Parse(opts.CloneAddr)
- if err != nil {
- return nil, err
- }
- u.User = nil
-
- fields := strings.Split(strings.Trim(u.Path, "/"), "/")
- if len(fields) != 2 {
- return nil, fmt.Errorf("invalid path: %s", u.Path)
- }
- project := fields[0]
- repoName := strings.TrimSuffix(fields[1], ".git")
-
- log.Trace("Create Codebase downloader. BaseURL: %v RepoName: %s", u, repoName)
-
- return NewCodebaseDownloader(ctx, u, project, repoName, opts.AuthUsername, opts.AuthPassword), nil
- }
-
- // GitServiceType returns the type of git service
- func (f *CodebaseDownloaderFactory) GitServiceType() structs.GitServiceType {
- return structs.CodebaseService
- }
-
- type codebaseUser struct {
- ID int64 `json:"id"`
- Name string `json:"name"`
- Email string `json:"email"`
- }
-
- // CodebaseDownloader implements a Downloader interface to get repository information
- // from Codebase
- type CodebaseDownloader struct {
- base.NullDownloader
- ctx context.Context
- client *http.Client
- baseURL *url.URL
- projectURL *url.URL
- project string
- repoName string
- maxIssueIndex int64
- userMap map[int64]*codebaseUser
- commitMap map[string]string
- }
-
- // SetContext set context
- func (d *CodebaseDownloader) SetContext(ctx context.Context) {
- d.ctx = ctx
- }
-
- // NewCodebaseDownloader creates a new downloader
- func NewCodebaseDownloader(ctx context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader {
- baseURL, _ := url.Parse("https://api3.codebasehq.com")
-
- downloader := &CodebaseDownloader{
- ctx: ctx,
- baseURL: baseURL,
- projectURL: projectURL,
- project: project,
- repoName: repoName,
- client: &http.Client{
- Transport: &http.Transport{
- Proxy: func(req *http.Request) (*url.URL, error) {
- if len(username) > 0 && len(password) > 0 {
- req.SetBasicAuth(username, password)
- }
- return proxy.Proxy()(req)
- },
- },
- },
- userMap: make(map[int64]*codebaseUser),
- commitMap: make(map[string]string),
- }
-
- return downloader
- }
-
- // FormatCloneURL add authentication into remote URLs
- func (d *CodebaseDownloader) FormatCloneURL(opts base.MigrateOptions, remoteAddr string) (string, error) {
- return opts.CloneAddr, nil
- }
-
- func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error {
- u, err := d.baseURL.Parse(endpoint)
- if err != nil {
- return err
- }
-
- if parameter != nil {
- query := u.Query()
- for k, v := range parameter {
- query.Set(k, v)
- }
- u.RawQuery = query.Encode()
- }
-
- req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil)
- if err != nil {
- return err
- }
- req.Header.Add("Accept", "application/xml")
-
- resp, err := d.client.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
-
- return xml.NewDecoder(resp.Body).Decode(&result)
- }
-
- // GetRepoInfo returns repository information
- // https://support.codebasehq.com/kb/projects
- func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) {
- var rawRepository struct {
- XMLName xml.Name `xml:"repository"`
- Name string `xml:"name"`
- Description string `xml:"description"`
- Permalink string `xml:"permalink"`
- CloneURL string `xml:"clone-url"`
- Source string `xml:"source"`
- }
-
- err := d.callAPI(
- fmt.Sprintf("/%s/%s", d.project, d.repoName),
- nil,
- &rawRepository,
- )
- if err != nil {
- return nil, err
- }
-
- return &base.Repository{
- Name: rawRepository.Name,
- Description: rawRepository.Description,
- CloneURL: rawRepository.CloneURL,
- OriginalURL: d.projectURL.String(),
- }, nil
- }
-
- // GetMilestones returns milestones
- // https://support.codebasehq.com/kb/tickets-and-milestones/milestones
- func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) {
- var rawMilestones struct {
- XMLName xml.Name `xml:"ticketing-milestone"`
- Type string `xml:"type,attr"`
- TicketingMilestone []struct {
- Text string `xml:",chardata"`
- ID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"id"`
- Identifier string `xml:"identifier"`
- Name string `xml:"name"`
- Deadline struct {
- Value string `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"deadline"`
- Description string `xml:"description"`
- Status string `xml:"status"`
- } `xml:"ticketing-milestone"`
- }
-
- err := d.callAPI(
- fmt.Sprintf("/%s/milestones", d.project),
- nil,
- &rawMilestones,
- )
- if err != nil {
- return nil, err
- }
-
- milestones := make([]*base.Milestone, 0, len(rawMilestones.TicketingMilestone))
- for _, milestone := range rawMilestones.TicketingMilestone {
- var deadline *time.Time
- if len(milestone.Deadline.Value) > 0 {
- if val, err := time.Parse("2006-01-02", milestone.Deadline.Value); err == nil {
- deadline = &val
- }
- }
-
- closed := deadline
- state := "closed"
- if milestone.Status == "active" {
- closed = nil
- state = ""
- }
-
- milestones = append(milestones, &base.Milestone{
- Title: milestone.Name,
- Deadline: deadline,
- Closed: closed,
- State: state,
- })
- }
- return milestones, nil
- }
-
- // GetLabels returns labels
- // https://support.codebasehq.com/kb/tickets-and-milestones/statuses-priorities-and-categories
- func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) {
- var rawTypes struct {
- XMLName xml.Name `xml:"ticketing-types"`
- Type string `xml:"type,attr"`
- TicketingType []struct {
- ID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"id"`
- Name string `xml:"name"`
- } `xml:"ticketing-type"`
- }
-
- err := d.callAPI(
- fmt.Sprintf("/%s/tickets/types", d.project),
- nil,
- &rawTypes,
- )
- if err != nil {
- return nil, err
- }
-
- labels := make([]*base.Label, 0, len(rawTypes.TicketingType))
- for _, label := range rawTypes.TicketingType {
- labels = append(labels, &base.Label{
- Name: label.Name,
- Color: "ffffff",
- })
- }
- return labels, nil
- }
-
- type codebaseIssueContext struct {
- Comments []*base.Comment
- }
-
- // GetIssues returns issues, limits are not supported
- // https://support.codebasehq.com/kb/tickets-and-milestones
- // https://support.codebasehq.com/kb/tickets-and-milestones/updating-tickets
- func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
- var rawIssues struct {
- XMLName xml.Name `xml:"tickets"`
- Type string `xml:"type,attr"`
- Ticket []struct {
- TicketID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"ticket-id"`
- Summary string `xml:"summary"`
- TicketType string `xml:"ticket-type"`
- ReporterID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"reporter-id"`
- Reporter string `xml:"reporter"`
- Type struct {
- Name string `xml:"name"`
- } `xml:"type"`
- Status struct {
- TreatAsClosed struct {
- Value bool `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"treat-as-closed"`
- } `xml:"status"`
- Milestone struct {
- Name string `xml:"name"`
- } `xml:"milestone"`
- UpdatedAt struct {
- Value time.Time `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"updated-at"`
- CreatedAt struct {
- Value time.Time `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"created-at"`
- } `xml:"ticket"`
- }
-
- err := d.callAPI(
- fmt.Sprintf("/%s/tickets", d.project),
- nil,
- &rawIssues,
- )
- if err != nil {
- return nil, false, err
- }
-
- issues := make([]*base.Issue, 0, len(rawIssues.Ticket))
- for _, issue := range rawIssues.Ticket {
- var notes struct {
- XMLName xml.Name `xml:"ticket-notes"`
- Type string `xml:"type,attr"`
- TicketNote []struct {
- Content string `xml:"content"`
- CreatedAt struct {
- Value time.Time `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"created-at"`
- UpdatedAt struct {
- Value time.Time `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"updated-at"`
- ID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"id"`
- UserID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"user-id"`
- } `xml:"ticket-note"`
- }
- err := d.callAPI(
- fmt.Sprintf("/%s/tickets/%d/notes", d.project, issue.TicketID.Value),
- nil,
- ¬es,
- )
- if err != nil {
- return nil, false, err
- }
- comments := make([]*base.Comment, 0, len(notes.TicketNote))
- for _, note := range notes.TicketNote {
- if len(note.Content) == 0 {
- continue
- }
- poster := d.tryGetUser(note.UserID.Value)
- comments = append(comments, &base.Comment{
- IssueIndex: issue.TicketID.Value,
- Index: note.ID.Value,
- PosterID: poster.ID,
- PosterName: poster.Name,
- PosterEmail: poster.Email,
- Content: note.Content,
- Created: note.CreatedAt.Value,
- Updated: note.UpdatedAt.Value,
- })
- }
- if len(comments) == 0 {
- comments = append(comments, &base.Comment{})
- }
-
- state := "open"
- if issue.Status.TreatAsClosed.Value {
- state = "closed"
- }
- poster := d.tryGetUser(issue.ReporterID.Value)
- issues = append(issues, &base.Issue{
- Title: issue.Summary,
- Number: issue.TicketID.Value,
- PosterName: poster.Name,
- PosterEmail: poster.Email,
- Content: comments[0].Content,
- Milestone: issue.Milestone.Name,
- State: state,
- Created: issue.CreatedAt.Value,
- Updated: issue.UpdatedAt.Value,
- Labels: []*base.Label{
- {Name: issue.Type.Name},
- },
- ForeignIndex: issue.TicketID.Value,
- Context: codebaseIssueContext{
- Comments: comments[1:],
- },
- })
-
- if d.maxIssueIndex < issue.TicketID.Value {
- d.maxIssueIndex = issue.TicketID.Value
- }
- }
-
- return issues, true, nil
- }
-
- // GetComments returns comments
- func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) {
- context, ok := commentable.GetContext().(codebaseIssueContext)
- if !ok {
- return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext())
- }
-
- return context.Comments, true, nil
- }
-
- // GetPullRequests returns pull requests
- // https://support.codebasehq.com/kb/repositories/merge-requests
- func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
- var rawMergeRequests struct {
- XMLName xml.Name `xml:"merge-requests"`
- Type string `xml:"type,attr"`
- MergeRequest []struct {
- ID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"id"`
- } `xml:"merge-request"`
- }
-
- err := d.callAPI(
- fmt.Sprintf("/%s/%s/merge_requests", d.project, d.repoName),
- map[string]string{
- "query": `"Target Project" is "` + d.repoName + `"`,
- "offset": strconv.Itoa((page - 1) * perPage),
- "count": strconv.Itoa(perPage),
- },
- &rawMergeRequests,
- )
- if err != nil {
- return nil, false, err
- }
-
- pullRequests := make([]*base.PullRequest, 0, len(rawMergeRequests.MergeRequest))
- for i, mr := range rawMergeRequests.MergeRequest {
- var rawMergeRequest struct {
- XMLName xml.Name `xml:"merge-request"`
- ID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"id"`
- SourceRef string `xml:"source-ref"`
- TargetRef string `xml:"target-ref"`
- Subject string `xml:"subject"`
- Status string `xml:"status"`
- UserID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"user-id"`
- CreatedAt struct {
- Value time.Time `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"created-at"`
- UpdatedAt struct {
- Value time.Time `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"updated-at"`
- Comments struct {
- Type string `xml:"type,attr"`
- Comment []struct {
- Content string `xml:"content"`
- ID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"id"`
- UserID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"user-id"`
- Action struct {
- Value string `xml:",chardata"`
- Nil string `xml:"nil,attr"`
- } `xml:"action"`
- CreatedAt struct {
- Value time.Time `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"created-at"`
- } `xml:"comment"`
- } `xml:"comments"`
- }
- err := d.callAPI(
- fmt.Sprintf("/%s/%s/merge_requests/%d", d.project, d.repoName, mr.ID.Value),
- nil,
- &rawMergeRequest,
- )
- if err != nil {
- return nil, false, err
- }
-
- number := d.maxIssueIndex + int64(i) + 1
-
- state := "open"
- merged := false
- var closeTime *time.Time
- var mergedTime *time.Time
- if rawMergeRequest.Status != "new" {
- state = "closed"
- closeTime = &rawMergeRequest.UpdatedAt.Value
- }
-
- comments := make([]*base.Comment, 0, len(rawMergeRequest.Comments.Comment))
- for _, comment := range rawMergeRequest.Comments.Comment {
- if len(comment.Content) == 0 {
- if comment.Action.Value == "merging" {
- merged = true
- mergedTime = &comment.CreatedAt.Value
- }
- continue
- }
- poster := d.tryGetUser(comment.UserID.Value)
- comments = append(comments, &base.Comment{
- IssueIndex: number,
- Index: comment.ID.Value,
- PosterID: poster.ID,
- PosterName: poster.Name,
- PosterEmail: poster.Email,
- Content: comment.Content,
- Created: comment.CreatedAt.Value,
- Updated: comment.CreatedAt.Value,
- })
- }
- if len(comments) == 0 {
- comments = append(comments, &base.Comment{})
- }
-
- poster := d.tryGetUser(rawMergeRequest.UserID.Value)
-
- pullRequests = append(pullRequests, &base.PullRequest{
- Title: rawMergeRequest.Subject,
- Number: number,
- PosterName: poster.Name,
- PosterEmail: poster.Email,
- Content: comments[0].Content,
- State: state,
- Created: rawMergeRequest.CreatedAt.Value,
- Updated: rawMergeRequest.UpdatedAt.Value,
- Closed: closeTime,
- Merged: merged,
- MergedTime: mergedTime,
- Head: base.PullRequestBranch{
- Ref: rawMergeRequest.SourceRef,
- SHA: d.getHeadCommit(rawMergeRequest.SourceRef),
- RepoName: d.repoName,
- },
- Base: base.PullRequestBranch{
- Ref: rawMergeRequest.TargetRef,
- SHA: d.getHeadCommit(rawMergeRequest.TargetRef),
- RepoName: d.repoName,
- },
- ForeignIndex: rawMergeRequest.ID.Value,
- Context: codebaseIssueContext{
- Comments: comments[1:],
- },
- })
- }
-
- return pullRequests, true, nil
- }
-
- func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser {
- if len(d.userMap) == 0 {
- var rawUsers struct {
- XMLName xml.Name `xml:"users"`
- Type string `xml:"type,attr"`
- User []struct {
- EmailAddress string `xml:"email-address"`
- ID struct {
- Value int64 `xml:",chardata"`
- Type string `xml:"type,attr"`
- } `xml:"id"`
- LastName string `xml:"last-name"`
- FirstName string `xml:"first-name"`
- Username string `xml:"username"`
- } `xml:"user"`
- }
-
- err := d.callAPI(
- "/users",
- nil,
- &rawUsers,
- )
- if err == nil {
- for _, user := range rawUsers.User {
- d.userMap[user.ID.Value] = &codebaseUser{
- Name: user.Username,
- Email: user.EmailAddress,
- }
- }
- }
- }
-
- user, ok := d.userMap[userID]
- if !ok {
- user = &codebaseUser{
- Name: fmt.Sprintf("User %d", userID),
- }
- d.userMap[userID] = user
- }
-
- return user
- }
-
- func (d *CodebaseDownloader) getHeadCommit(ref string) string {
- commitRef, ok := d.commitMap[ref]
- if !ok {
- var rawCommits struct {
- XMLName xml.Name `xml:"commits"`
- Type string `xml:"type,attr"`
- Commit []struct {
- Ref string `xml:"ref"`
- } `xml:"commit"`
- }
- err := d.callAPI(
- fmt.Sprintf("/%s/%s/commits/%s", d.project, d.repoName, ref),
- nil,
- &rawCommits,
- )
- if err == nil && len(rawCommits.Commit) > 0 {
- commitRef = rawCommits.Commit[0].Ref
- d.commitMap[ref] = commitRef
- }
- }
- return commitRef
- }
|