123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634 |
- // Copyright 2021 The Gitea Authors. All rights reserved.
- // SPDX-License-Identifier: MIT
-
- package migrations
-
- import (
- "context"
- "fmt"
- "net/http"
- "net/url"
- "strconv"
- "strings"
- "time"
-
- "code.gitea.io/gitea/modules/json"
- "code.gitea.io/gitea/modules/log"
- base "code.gitea.io/gitea/modules/migration"
- "code.gitea.io/gitea/modules/structs"
- )
-
- var (
- _ base.Downloader = &OneDevDownloader{}
- _ base.DownloaderFactory = &OneDevDownloaderFactory{}
- )
-
- func init() {
- RegisterDownloaderFactory(&OneDevDownloaderFactory{})
- }
-
- // OneDevDownloaderFactory defines a downloader factory
- type OneDevDownloaderFactory struct{}
-
- // New returns a downloader related to this factory according MigrateOptions
- func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
- u, err := url.Parse(opts.CloneAddr)
- if err != nil {
- return nil, err
- }
-
- var repoName string
-
- fields := strings.Split(strings.Trim(u.Path, "/"), "/")
- if len(fields) == 2 && fields[0] == "projects" {
- repoName = fields[1]
- } else if len(fields) == 1 {
- repoName = fields[0]
- } else {
- return nil, fmt.Errorf("invalid path: %s", u.Path)
- }
-
- u.Path = ""
- u.Fragment = ""
-
- log.Trace("Create onedev downloader. BaseURL: %v RepoName: %s", u, repoName)
-
- return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoName), nil
- }
-
- // GitServiceType returns the type of git service
- func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType {
- return structs.OneDevService
- }
-
- type onedevUser struct {
- ID int64 `json:"id"`
- Name string `json:"name"`
- Email string `json:"email"`
- }
-
- // OneDevDownloader implements a Downloader interface to get repository information
- // from OneDev
- type OneDevDownloader struct {
- base.NullDownloader
- ctx context.Context
- client *http.Client
- baseURL *url.URL
- repoName string
- repoID int64
- maxIssueIndex int64
- userMap map[int64]*onedevUser
- milestoneMap map[int64]string
- }
-
- // SetContext set context
- func (d *OneDevDownloader) SetContext(ctx context.Context) {
- d.ctx = ctx
- }
-
- // NewOneDevDownloader creates a new downloader
- func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader {
- downloader := &OneDevDownloader{
- ctx: ctx,
- baseURL: baseURL,
- repoName: repoName,
- client: &http.Client{
- Transport: &http.Transport{
- Proxy: func(req *http.Request) (*url.URL, error) {
- if len(username) > 0 && len(password) > 0 {
- req.SetBasicAuth(username, password)
- }
- return nil, nil
- },
- },
- },
- userMap: make(map[int64]*onedevUser),
- milestoneMap: make(map[int64]string),
- }
-
- return downloader
- }
-
- // String implements Stringer
- func (d *OneDevDownloader) String() string {
- return fmt.Sprintf("migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoName)
- }
-
- func (d *OneDevDownloader) LogString() string {
- if d == nil {
- return "<OneDevDownloader nil>"
- }
- return fmt.Sprintf("<OneDevDownloader %s [%d]/%s>", d.baseURL, d.repoID, d.repoName)
- }
-
- func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result any) error {
- u, err := d.baseURL.Parse(endpoint)
- if err != nil {
- return err
- }
-
- if parameter != nil {
- query := u.Query()
- for k, v := range parameter {
- query.Set(k, v)
- }
- u.RawQuery = query.Encode()
- }
-
- req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil)
- if err != nil {
- return err
- }
-
- resp, err := d.client.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
-
- decoder := json.NewDecoder(resp.Body)
- return decoder.Decode(&result)
- }
-
- // GetRepoInfo returns repository information
- func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) {
- info := make([]struct {
- ID int64 `json:"id"`
- Name string `json:"name"`
- Description string `json:"description"`
- }, 0, 1)
-
- err := d.callAPI(
- "/api/projects",
- map[string]string{
- "query": `"Name" is "` + d.repoName + `"`,
- "offset": "0",
- "count": "1",
- },
- &info,
- )
- if err != nil {
- return nil, err
- }
- if len(info) != 1 {
- return nil, fmt.Errorf("Project %s not found", d.repoName)
- }
-
- d.repoID = info[0].ID
-
- cloneURL, err := d.baseURL.Parse(info[0].Name)
- if err != nil {
- return nil, err
- }
- originalURL, err := d.baseURL.Parse("/projects/" + info[0].Name)
- if err != nil {
- return nil, err
- }
-
- return &base.Repository{
- Name: info[0].Name,
- Description: info[0].Description,
- CloneURL: cloneURL.String(),
- OriginalURL: originalURL.String(),
- }, nil
- }
-
- // GetMilestones returns milestones
- func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) {
- rawMilestones := make([]struct {
- ID int64 `json:"id"`
- Name string `json:"name"`
- Description string `json:"description"`
- DueDate *time.Time `json:"dueDate"`
- Closed bool `json:"closed"`
- }, 0, 100)
-
- endpoint := fmt.Sprintf("/api/projects/%d/milestones", d.repoID)
-
- milestones := make([]*base.Milestone, 0, 100)
- offset := 0
- for {
- err := d.callAPI(
- endpoint,
- map[string]string{
- "offset": strconv.Itoa(offset),
- "count": "100",
- },
- &rawMilestones,
- )
- if err != nil {
- return nil, err
- }
- if len(rawMilestones) == 0 {
- break
- }
- offset += 100
-
- for _, milestone := range rawMilestones {
- d.milestoneMap[milestone.ID] = milestone.Name
- closed := milestone.DueDate
- if !milestone.Closed {
- closed = nil
- }
-
- milestones = append(milestones, &base.Milestone{
- Title: milestone.Name,
- Description: milestone.Description,
- Deadline: milestone.DueDate,
- Closed: closed,
- })
- }
- }
- return milestones, nil
- }
-
- // GetLabels returns labels
- func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) {
- return []*base.Label{
- {
- Name: "Bug",
- Color: "f64e60",
- },
- {
- Name: "Build Failure",
- Color: "f64e60",
- },
- {
- Name: "Discussion",
- Color: "8950fc",
- },
- {
- Name: "Improvement",
- Color: "1bc5bd",
- },
- {
- Name: "New Feature",
- Color: "1bc5bd",
- },
- {
- Name: "Support Request",
- Color: "8950fc",
- },
- }, nil
- }
-
- type onedevIssueContext struct {
- IsPullRequest bool
- }
-
- // GetIssues returns issues
- func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
- rawIssues := make([]struct {
- ID int64 `json:"id"`
- Number int64 `json:"number"`
- State string `json:"state"`
- Title string `json:"title"`
- Description string `json:"description"`
- SubmitterID int64 `json:"submitterId"`
- SubmitDate time.Time `json:"submitDate"`
- }, 0, perPage)
-
- err := d.callAPI(
- "/api/issues",
- map[string]string{
- "query": `"Project" is "` + d.repoName + `"`,
- "offset": strconv.Itoa((page - 1) * perPage),
- "count": strconv.Itoa(perPage),
- },
- &rawIssues,
- )
- if err != nil {
- return nil, false, err
- }
-
- issues := make([]*base.Issue, 0, len(rawIssues))
- for _, issue := range rawIssues {
- fields := make([]struct {
- Name string `json:"name"`
- Value string `json:"value"`
- }, 0, 10)
- err := d.callAPI(
- fmt.Sprintf("/api/issues/%d/fields", issue.ID),
- nil,
- &fields,
- )
- if err != nil {
- return nil, false, err
- }
-
- var label *base.Label
- for _, field := range fields {
- if field.Name == "Type" {
- label = &base.Label{Name: field.Value}
- break
- }
- }
-
- milestones := make([]struct {
- ID int64 `json:"id"`
- Name string `json:"name"`
- }, 0, 10)
- err = d.callAPI(
- fmt.Sprintf("/api/issues/%d/milestones", issue.ID),
- nil,
- &milestones,
- )
- if err != nil {
- return nil, false, err
- }
- milestoneID := int64(0)
- if len(milestones) > 0 {
- milestoneID = milestones[0].ID
- }
-
- state := strings.ToLower(issue.State)
- if state == "released" {
- state = "closed"
- }
- poster := d.tryGetUser(issue.SubmitterID)
- issues = append(issues, &base.Issue{
- Title: issue.Title,
- Number: issue.Number,
- PosterName: poster.Name,
- PosterEmail: poster.Email,
- Content: issue.Description,
- Milestone: d.milestoneMap[milestoneID],
- State: state,
- Created: issue.SubmitDate,
- Updated: issue.SubmitDate,
- Labels: []*base.Label{label},
- ForeignIndex: issue.ID,
- Context: onedevIssueContext{IsPullRequest: false},
- })
-
- if d.maxIssueIndex < issue.Number {
- d.maxIssueIndex = issue.Number
- }
- }
-
- return issues, len(issues) == 0, nil
- }
-
- // GetComments returns comments
- func (d *OneDevDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) {
- context, ok := commentable.GetContext().(onedevIssueContext)
- if !ok {
- return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext())
- }
-
- rawComments := make([]struct {
- ID int64 `json:"id"`
- Date time.Time `json:"date"`
- UserID int64 `json:"userId"`
- Content string `json:"content"`
- }, 0, 100)
-
- var endpoint string
- if context.IsPullRequest {
- endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", commentable.GetForeignIndex())
- } else {
- endpoint = fmt.Sprintf("/api/issues/%d/comments", commentable.GetForeignIndex())
- }
-
- err := d.callAPI(
- endpoint,
- nil,
- &rawComments,
- )
- if err != nil {
- return nil, false, err
- }
-
- rawChanges := make([]struct {
- Date time.Time `json:"date"`
- UserID int64 `json:"userId"`
- Data map[string]any `json:"data"`
- }, 0, 100)
-
- if context.IsPullRequest {
- endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", commentable.GetForeignIndex())
- } else {
- endpoint = fmt.Sprintf("/api/issues/%d/changes", commentable.GetForeignIndex())
- }
-
- err = d.callAPI(
- endpoint,
- nil,
- &rawChanges,
- )
- if err != nil {
- return nil, false, err
- }
-
- comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges))
- for _, comment := range rawComments {
- if len(comment.Content) == 0 {
- continue
- }
- poster := d.tryGetUser(comment.UserID)
- comments = append(comments, &base.Comment{
- IssueIndex: commentable.GetLocalIndex(),
- Index: comment.ID,
- PosterID: poster.ID,
- PosterName: poster.Name,
- PosterEmail: poster.Email,
- Content: comment.Content,
- Created: comment.Date,
- Updated: comment.Date,
- })
- }
- for _, change := range rawChanges {
- contentV, ok := change.Data["content"]
- if !ok {
- contentV, ok = change.Data["comment"]
- if !ok {
- continue
- }
- }
- content, ok := contentV.(string)
- if !ok || len(content) == 0 {
- continue
- }
-
- poster := d.tryGetUser(change.UserID)
- comments = append(comments, &base.Comment{
- IssueIndex: commentable.GetLocalIndex(),
- PosterID: poster.ID,
- PosterName: poster.Name,
- PosterEmail: poster.Email,
- Content: content,
- Created: change.Date,
- Updated: change.Date,
- })
- }
-
- return comments, true, nil
- }
-
- // GetPullRequests returns pull requests
- func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
- rawPullRequests := make([]struct {
- ID int64 `json:"id"`
- Number int64 `json:"number"`
- Title string `json:"title"`
- SubmitterID int64 `json:"submitterId"`
- SubmitDate time.Time `json:"submitDate"`
- Description string `json:"description"`
- TargetBranch string `json:"targetBranch"`
- SourceBranch string `json:"sourceBranch"`
- BaseCommitHash string `json:"baseCommitHash"`
- CloseInfo *struct {
- Date *time.Time `json:"date"`
- Status string `json:"status"`
- }
- }, 0, perPage)
-
- err := d.callAPI(
- "/api/pull-requests",
- map[string]string{
- "query": `"Target Project" is "` + d.repoName + `"`,
- "offset": strconv.Itoa((page - 1) * perPage),
- "count": strconv.Itoa(perPage),
- },
- &rawPullRequests,
- )
- if err != nil {
- return nil, false, err
- }
-
- pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests))
- for _, pr := range rawPullRequests {
- var mergePreview struct {
- TargetHeadCommitHash string `json:"targetHeadCommitHash"`
- HeadCommitHash string `json:"headCommitHash"`
- MergeStrategy string `json:"mergeStrategy"`
- MergeCommitHash string `json:"mergeCommitHash"`
- }
- err := d.callAPI(
- fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID),
- nil,
- &mergePreview,
- )
- if err != nil {
- return nil, false, err
- }
-
- state := "open"
- merged := false
- var closeTime *time.Time
- var mergedTime *time.Time
- if pr.CloseInfo != nil {
- state = "closed"
- closeTime = pr.CloseInfo.Date
- if pr.CloseInfo.Status == "MERGED" { // "DISCARDED"
- merged = true
- mergedTime = pr.CloseInfo.Date
- }
- }
- poster := d.tryGetUser(pr.SubmitterID)
-
- number := pr.Number + d.maxIssueIndex
- pullRequests = append(pullRequests, &base.PullRequest{
- Title: pr.Title,
- Number: number,
- PosterName: poster.Name,
- PosterID: poster.ID,
- Content: pr.Description,
- State: state,
- Created: pr.SubmitDate,
- Updated: pr.SubmitDate,
- Closed: closeTime,
- Merged: merged,
- MergedTime: mergedTime,
- Head: base.PullRequestBranch{
- Ref: pr.SourceBranch,
- SHA: mergePreview.HeadCommitHash,
- RepoName: d.repoName,
- },
- Base: base.PullRequestBranch{
- Ref: pr.TargetBranch,
- SHA: mergePreview.TargetHeadCommitHash,
- RepoName: d.repoName,
- },
- ForeignIndex: pr.ID,
- Context: onedevIssueContext{IsPullRequest: true},
- })
-
- // SECURITY: Ensure that the PR is safe
- _ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d)
- }
-
- return pullRequests, len(pullRequests) == 0, nil
- }
-
- // GetReviews returns pull requests reviews
- func (d *OneDevDownloader) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) {
- rawReviews := make([]struct {
- ID int64 `json:"id"`
- UserID int64 `json:"userId"`
- Result *struct {
- Commit string `json:"commit"`
- Approved bool `json:"approved"`
- Comment string `json:"comment"`
- }
- }, 0, 100)
-
- err := d.callAPI(
- fmt.Sprintf("/api/pull-requests/%d/reviews", reviewable.GetForeignIndex()),
- nil,
- &rawReviews,
- )
- if err != nil {
- return nil, err
- }
-
- reviews := make([]*base.Review, 0, len(rawReviews))
- for _, review := range rawReviews {
- state := base.ReviewStatePending
- content := ""
- if review.Result != nil {
- if len(review.Result.Comment) > 0 {
- state = base.ReviewStateCommented
- content = review.Result.Comment
- }
- if review.Result.Approved {
- state = base.ReviewStateApproved
- }
- }
-
- poster := d.tryGetUser(review.UserID)
- reviews = append(reviews, &base.Review{
- IssueIndex: reviewable.GetLocalIndex(),
- ReviewerID: poster.ID,
- ReviewerName: poster.Name,
- Content: content,
- State: state,
- })
- }
-
- return reviews, nil
- }
-
- // GetTopics return repository topics
- func (d *OneDevDownloader) GetTopics() ([]string, error) {
- return []string{}, nil
- }
-
- func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser {
- user, ok := d.userMap[userID]
- if !ok {
- err := d.callAPI(
- fmt.Sprintf("/api/users/%d", userID),
- nil,
- &user,
- )
- if err != nil {
- user = &onedevUser{
- Name: fmt.Sprintf("User %d", userID),
- }
- }
- d.userMap[userID] = user
- }
-
- return user
- }
|