"code.gitea.io/gitea/modules/convert"
"code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations"
- "code.gitea.io/gitea/modules/migrations/base"
+ base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
+ "code.gitea.io/gitea/services/migrations"
"github.com/urfave/cli"
)
"os"
"testing"
- "code.gitea.io/gitea/models/unittest"
-
"code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/migrations"
+ "code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/services/migrations"
"github.com/stretchr/testify/assert"
)
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/git"
- migration "code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/repository"
mirror_service "code.gitea.io/gitea/services/mirror"
release_service "code.gitea.io/gitea/services/release"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/json"
- migration "code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/secret"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "time"
+
+// Comment is a standard comment information
+type Comment struct {
+ IssueIndex int64 `yaml:"issue_index"`
+ PosterID int64 `yaml:"poster_id"`
+ PosterName string `yaml:"poster_name"`
+ PosterEmail string `yaml:"poster_email"`
+ Created time.Time
+ Updated time.Time
+ Content string
+ Reactions []*Reaction
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+ "context"
+
+ "code.gitea.io/gitea/modules/structs"
+)
+
+// GetCommentOptions represents an options for get comment
+type GetCommentOptions struct {
+ Context IssueContext
+ Page int
+ PageSize int
+}
+
+// Downloader downloads the site repo information
+type Downloader interface {
+ SetContext(context.Context)
+ GetRepoInfo() (*Repository, error)
+ GetTopics() ([]string, error)
+ GetMilestones() ([]*Milestone, error)
+ GetReleases() ([]*Release, error)
+ GetLabels() ([]*Label, error)
+ GetIssues(page, perPage int) ([]*Issue, bool, error)
+ GetComments(opts GetCommentOptions) ([]*Comment, bool, error)
+ SupportGetRepoComments() bool
+ GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
+ GetReviews(pullRequestContext IssueContext) ([]*Review, error)
+ FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
+}
+
+// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
+type DownloaderFactory interface {
+ New(ctx context.Context, opts MigrateOptions) (Downloader, error)
+ GitServiceType() structs.GitServiceType
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "fmt"
+
+// ErrNotSupported represents status if a downloader do not supported something.
+type ErrNotSupported struct {
+ Entity string
+}
+
+// IsErrNotSupported checks if an error is an ErrNotSupported
+func IsErrNotSupported(err error) bool {
+ _, ok := err.(ErrNotSupported)
+ return ok
+}
+
+// Error return error message
+func (err ErrNotSupported) Error() string {
+ if len(err.Entity) != 0 {
+ return fmt.Sprintf("'%s' not supported", err.Entity)
+ }
+ return "not supported"
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "time"
+
+// IssueContext is used to map between local and foreign issue/PR ids.
+type IssueContext interface {
+ LocalID() int64
+ ForeignID() int64
+}
+
+// BasicIssueContext is a 1:1 mapping between local and foreign ids.
+type BasicIssueContext int64
+
+// LocalID gets the local id.
+func (c BasicIssueContext) LocalID() int64 {
+ return int64(c)
+}
+
+// ForeignID gets the foreign id.
+func (c BasicIssueContext) ForeignID() int64 {
+ return int64(c)
+}
+
+// Issue is a standard issue information
+type Issue struct {
+ Number int64
+ PosterID int64 `yaml:"poster_id"`
+ PosterName string `yaml:"poster_name"`
+ PosterEmail string `yaml:"poster_email"`
+ Title string
+ Content string
+ Ref string
+ Milestone string
+ State string // closed, open
+ IsLocked bool `yaml:"is_locked"`
+ Created time.Time
+ Updated time.Time
+ Closed *time.Time
+ Labels []*Label
+ Reactions []*Reaction
+ Assignees []string
+ Context IssueContext `yaml:"-"`
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+// Label defines a standard label information
+type Label struct {
+ Name string
+ Color string
+ Description string
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+// Messenger is a formatting function similar to i18n.Tr
+type Messenger func(key string, args ...interface{})
+
+// NilMessenger represents an empty formatting function
+func NilMessenger(string, ...interface{}) {}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "time"
+
+// Milestone defines a standard milestone
+type Milestone struct {
+ Title string
+ Description string
+ Deadline *time.Time
+ Created time.Time
+ Updated *time.Time
+ Closed *time.Time
+ State string // open, closed
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+ "context"
+ "net/url"
+)
+
+// NullDownloader implements a blank downloader
+type NullDownloader struct {
+}
+
+var (
+ _ Downloader = &NullDownloader{}
+)
+
+// SetContext set context
+func (n NullDownloader) SetContext(_ context.Context) {}
+
+// GetRepoInfo returns a repository information
+func (n NullDownloader) GetRepoInfo() (*Repository, error) {
+ return nil, &ErrNotSupported{Entity: "RepoInfo"}
+}
+
+// GetTopics return repository topics
+func (n NullDownloader) GetTopics() ([]string, error) {
+ return nil, &ErrNotSupported{Entity: "Topics"}
+}
+
+// GetMilestones returns milestones
+func (n NullDownloader) GetMilestones() ([]*Milestone, error) {
+ return nil, &ErrNotSupported{Entity: "Milestones"}
+}
+
+// GetReleases returns releases
+func (n NullDownloader) GetReleases() ([]*Release, error) {
+ return nil, &ErrNotSupported{Entity: "Releases"}
+}
+
+// GetLabels returns labels
+func (n NullDownloader) GetLabels() ([]*Label, error) {
+ return nil, &ErrNotSupported{Entity: "Labels"}
+}
+
+// GetIssues returns issues according start and limit
+func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
+ return nil, false, &ErrNotSupported{Entity: "Issues"}
+}
+
+// GetComments returns comments according the options
+func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) {
+ return nil, false, &ErrNotSupported{Entity: "Comments"}
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
+ return nil, false, &ErrNotSupported{Entity: "PullRequests"}
+}
+
+// GetReviews returns pull requests review
+func (n NullDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
+ return nil, &ErrNotSupported{Entity: "Reviews"}
+}
+
+// FormatCloneURL add authentification into remote URLs
+func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
+ if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
+ u, err := url.Parse(remoteAddr)
+ if err != nil {
+ return "", err
+ }
+ u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
+ if len(opts.AuthToken) > 0 {
+ u.User = url.UserPassword("oauth2", opts.AuthToken)
+ }
+ return u.String(), nil
+ }
+ return remoteAddr, nil
+}
+
+// SupportGetRepoComments return true if it supports get repo comments
+func (n NullDownloader) SupportGetRepoComments() bool {
+ return false
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "code.gitea.io/gitea/modules/structs"
+
+// MigrateOptions defines the way a repository gets migrated
+// this is for internal usage by migrations module and func who interact with it
+type MigrateOptions struct {
+ // required: true
+ CloneAddr string `json:"clone_addr" binding:"Required"`
+ CloneAddrEncrypted string `json:"clone_addr_encrypted,omitempty"`
+ AuthUsername string `json:"auth_username"`
+ AuthPassword string `json:"-"`
+ AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"`
+ AuthToken string `json:"-"`
+ AuthTokenEncrypted string `json:"auth_token_encrypted,omitempty"`
+ // required: true
+ UID int `json:"uid" binding:"Required"`
+ // required: true
+ RepoName string `json:"repo_name" binding:"Required"`
+ Mirror bool `json:"mirror"`
+ LFS bool `json:"lfs"`
+ LFSEndpoint string `json:"lfs_endpoint"`
+ Private bool `json:"private"`
+ Description string `json:"description"`
+ OriginalURL string
+ GitServiceType structs.GitServiceType
+ Wiki bool
+ Issues bool
+ Milestones bool
+ Labels bool
+ Releases bool
+ Comments bool
+ PullRequests bool
+ ReleaseAssets bool
+ MigrateToRepoID int64
+ MirrorInterval string `json:"mirror_interval"`
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+ "fmt"
+ "time"
+)
+
+// PullRequest defines a standard pull request information
+type PullRequest struct {
+ Number int64
+ Title string
+ PosterName string `yaml:"poster_name"`
+ PosterID int64 `yaml:"poster_id"`
+ PosterEmail string `yaml:"poster_email"`
+ Content string
+ Milestone string
+ State string
+ Created time.Time
+ Updated time.Time
+ Closed *time.Time
+ Labels []*Label
+ PatchURL string `yaml:"patch_url"`
+ Merged bool
+ MergedTime *time.Time `yaml:"merged_time"`
+ MergeCommitSHA string `yaml:"merge_commit_sha"`
+ Head PullRequestBranch
+ Base PullRequestBranch
+ Assignees []string
+ IsLocked bool `yaml:"is_locked"`
+ Reactions []*Reaction
+ Context IssueContext `yaml:"-"`
+}
+
+// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
+func (p *PullRequest) IsForkPullRequest() bool {
+ return p.Head.RepoPath() != p.Base.RepoPath()
+}
+
+// GetGitRefName returns pull request relative path to head
+func (p PullRequest) GetGitRefName() string {
+ return fmt.Sprintf("refs/pull/%d/head", p.Number)
+}
+
+// PullRequestBranch represents a pull request branch
+type PullRequestBranch struct {
+ CloneURL string `yaml:"clone_url"`
+ Ref string
+ SHA string
+ RepoName string `yaml:"repo_name"`
+ OwnerName string `yaml:"owner_name"`
+}
+
+// RepoPath returns pull request repo path
+func (p PullRequestBranch) RepoPath() string {
+ return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName)
+}
--- /dev/null
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+// Reaction represents a reaction to an issue/pr/comment.
+type Reaction struct {
+ UserID int64 `yaml:"user_id"`
+ UserName string `yaml:"user_name"`
+ Content string
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+ "io"
+ "time"
+)
+
+// ReleaseAsset represents a release asset
+type ReleaseAsset struct {
+ ID int64
+ Name string
+ ContentType *string `yaml:"content_type"`
+ Size *int
+ DownloadCount *int `yaml:"download_count"`
+ Created time.Time
+ Updated time.Time
+ DownloadURL *string `yaml:"download_url"`
+ // if DownloadURL is nil, the function should be invoked
+ DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
+}
+
+// Release represents a release
+type Release struct {
+ TagName string `yaml:"tag_name"`
+ TargetCommitish string `yaml:"target_commitish"`
+ Name string
+ Body string
+ Draft bool
+ Prerelease bool
+ PublisherID int64 `yaml:"publisher_id"`
+ PublisherName string `yaml:"publisher_name"`
+ PublisherEmail string `yaml:"publisher_email"`
+ Assets []*ReleaseAsset
+ Created time.Time
+ Published time.Time
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+// Repository defines a standard repository information
+type Repository struct {
+ Name string
+ Owner string
+ IsPrivate bool `yaml:"is_private"`
+ IsMirror bool `yaml:"is_mirror"`
+ Description string
+ CloneURL string `yaml:"clone_url"`
+ OriginalURL string `yaml:"original_url"`
+ DefaultBranch string
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+ "context"
+ "time"
+)
+
+var (
+ _ Downloader = &RetryDownloader{}
+)
+
+// RetryDownloader retry the downloads
+type RetryDownloader struct {
+ Downloader
+ ctx context.Context
+ RetryTimes int // the total execute times
+ RetryDelay int // time to delay seconds
+}
+
+// NewRetryDownloader creates a retry downloader
+func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
+ return &RetryDownloader{
+ Downloader: downloader,
+ ctx: ctx,
+ RetryTimes: retryTimes,
+ RetryDelay: retryDelay,
+ }
+}
+
+func (d *RetryDownloader) retry(work func() error) error {
+ var (
+ times = d.RetryTimes
+ err error
+ )
+ for ; times > 0; times-- {
+ if err = work(); err == nil {
+ return nil
+ }
+ if IsErrNotSupported(err) {
+ return err
+ }
+ select {
+ case <-d.ctx.Done():
+ return d.ctx.Err()
+ case <-time.After(time.Second * time.Duration(d.RetryDelay)):
+ }
+ }
+ return err
+}
+
+// SetContext set context
+func (d *RetryDownloader) SetContext(ctx context.Context) {
+ d.ctx = ctx
+ d.Downloader.SetContext(ctx)
+}
+
+// GetRepoInfo returns a repository information with retry
+func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
+ var (
+ repo *Repository
+ err error
+ )
+
+ err = d.retry(func() error {
+ repo, err = d.Downloader.GetRepoInfo()
+ return err
+ })
+
+ return repo, err
+}
+
+// GetTopics returns a repository's topics with retry
+func (d *RetryDownloader) GetTopics() ([]string, error) {
+ var (
+ topics []string
+ err error
+ )
+
+ err = d.retry(func() error {
+ topics, err = d.Downloader.GetTopics()
+ return err
+ })
+
+ return topics, err
+}
+
+// GetMilestones returns a repository's milestones with retry
+func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
+ var (
+ milestones []*Milestone
+ err error
+ )
+
+ err = d.retry(func() error {
+ milestones, err = d.Downloader.GetMilestones()
+ return err
+ })
+
+ return milestones, err
+}
+
+// GetReleases returns a repository's releases with retry
+func (d *RetryDownloader) GetReleases() ([]*Release, error) {
+ var (
+ releases []*Release
+ err error
+ )
+
+ err = d.retry(func() error {
+ releases, err = d.Downloader.GetReleases()
+ return err
+ })
+
+ return releases, err
+}
+
+// GetLabels returns a repository's labels with retry
+func (d *RetryDownloader) GetLabels() ([]*Label, error) {
+ var (
+ labels []*Label
+ err error
+ )
+
+ err = d.retry(func() error {
+ labels, err = d.Downloader.GetLabels()
+ return err
+ })
+
+ return labels, err
+}
+
+// GetIssues returns a repository's issues with retry
+func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
+ var (
+ issues []*Issue
+ isEnd bool
+ err error
+ )
+
+ err = d.retry(func() error {
+ issues, isEnd, err = d.Downloader.GetIssues(page, perPage)
+ return err
+ })
+
+ return issues, isEnd, err
+}
+
+// GetComments returns a repository's comments with retry
+func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) {
+ var (
+ comments []*Comment
+ isEnd bool
+ err error
+ )
+
+ err = d.retry(func() error {
+ comments, isEnd, err = d.Downloader.GetComments(opts)
+ return err
+ })
+
+ return comments, isEnd, err
+}
+
+// GetPullRequests returns a repository's pull requests with retry
+func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
+ var (
+ prs []*PullRequest
+ err error
+ isEnd bool
+ )
+
+ err = d.retry(func() error {
+ prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage)
+ return err
+ })
+
+ return prs, isEnd, err
+}
+
+// GetReviews returns pull requests reviews
+func (d *RetryDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
+ var (
+ reviews []*Review
+ err error
+ )
+
+ err = d.retry(func() error {
+ reviews, err = d.Downloader.GetReviews(pullRequestContext)
+ return err
+ })
+
+ return reviews, err
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import "time"
+
+// enumerate all review states
+const (
+ ReviewStatePending = "PENDING"
+ ReviewStateApproved = "APPROVED"
+ ReviewStateChangesRequested = "CHANGES_REQUESTED"
+ ReviewStateCommented = "COMMENTED"
+)
+
+// Review is a standard review information
+type Review struct {
+ ID int64
+ IssueIndex int64 `yaml:"issue_index"`
+ ReviewerID int64 `yaml:"reviewer_id"`
+ ReviewerName string `yaml:"reviewer_name"`
+ Official bool
+ CommitID string `yaml:"commit_id"`
+ Content string
+ CreatedAt time.Time `yaml:"created_at"`
+ State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
+ Comments []*ReviewComment
+}
+
+// ReviewComment represents a review comment
+type ReviewComment struct {
+ ID int64
+ InReplyTo int64 `yaml:"in_reply_to"`
+ Content string
+ TreePath string `yaml:"tree_path"`
+ DiffHunk string `yaml:"diff_hunk"`
+ Position int
+ Line int
+ CommitID string `yaml:"commit_id"`
+ PosterID int64 `yaml:"poster_id"`
+ Reactions []*Reaction
+ CreatedAt time.Time `yaml:"created_at"`
+ UpdatedAt time.Time `yaml:"updated_at"`
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+// Uploader uploads all the information of one repository
+type Uploader interface {
+ MaxBatchInsertSize(tp string) int
+ CreateRepo(repo *Repository, opts MigrateOptions) error
+ CreateTopics(topic ...string) error
+ CreateMilestones(milestones ...*Milestone) error
+ CreateReleases(releases ...*Release) error
+ SyncTags() error
+ CreateLabels(labels ...*Label) error
+ CreateIssues(issues ...*Issue) error
+ CreateComments(comments ...*Comment) error
+ CreatePullRequests(prs ...*PullRequest) error
+ CreateReviews(reviews ...*Review) error
+ Rollback() error
+ Finish() error
+ Close()
+}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import "time"
-
-// Comment is a standard comment information
-type Comment struct {
- IssueIndex int64 `yaml:"issue_index"`
- PosterID int64 `yaml:"poster_id"`
- PosterName string `yaml:"poster_name"`
- PosterEmail string `yaml:"poster_email"`
- Created time.Time
- Updated time.Time
- Content string
- Reactions []*Reaction
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import (
- "context"
-
- "code.gitea.io/gitea/modules/structs"
-)
-
-// GetCommentOptions represents an options for get comment
-type GetCommentOptions struct {
- Context IssueContext
- Page int
- PageSize int
-}
-
-// Downloader downloads the site repo information
-type Downloader interface {
- SetContext(context.Context)
- GetRepoInfo() (*Repository, error)
- GetTopics() ([]string, error)
- GetMilestones() ([]*Milestone, error)
- GetReleases() ([]*Release, error)
- GetLabels() ([]*Label, error)
- GetIssues(page, perPage int) ([]*Issue, bool, error)
- GetComments(opts GetCommentOptions) ([]*Comment, bool, error)
- SupportGetRepoComments() bool
- GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
- GetReviews(pullRequestContext IssueContext) ([]*Review, error)
- FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
-}
-
-// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
-type DownloaderFactory interface {
- New(ctx context.Context, opts MigrateOptions) (Downloader, error)
- GitServiceType() structs.GitServiceType
-}
+++ /dev/null
-// Copyright 2021 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import "fmt"
-
-// ErrNotSupported represents status if a downloader do not supported something.
-type ErrNotSupported struct {
- Entity string
-}
-
-// IsErrNotSupported checks if an error is an ErrNotSupported
-func IsErrNotSupported(err error) bool {
- _, ok := err.(ErrNotSupported)
- return ok
-}
-
-// Error return error message
-func (err ErrNotSupported) Error() string {
- if len(err.Entity) != 0 {
- return fmt.Sprintf("'%s' not supported", err.Entity)
- }
- return "not supported"
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import "time"
-
-// IssueContext is used to map between local and foreign issue/PR ids.
-type IssueContext interface {
- LocalID() int64
- ForeignID() int64
-}
-
-// BasicIssueContext is a 1:1 mapping between local and foreign ids.
-type BasicIssueContext int64
-
-// LocalID gets the local id.
-func (c BasicIssueContext) LocalID() int64 {
- return int64(c)
-}
-
-// ForeignID gets the foreign id.
-func (c BasicIssueContext) ForeignID() int64 {
- return int64(c)
-}
-
-// Issue is a standard issue information
-type Issue struct {
- Number int64
- PosterID int64 `yaml:"poster_id"`
- PosterName string `yaml:"poster_name"`
- PosterEmail string `yaml:"poster_email"`
- Title string
- Content string
- Ref string
- Milestone string
- State string // closed, open
- IsLocked bool `yaml:"is_locked"`
- Created time.Time
- Updated time.Time
- Closed *time.Time
- Labels []*Label
- Reactions []*Reaction
- Assignees []string
- Context IssueContext `yaml:"-"`
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-// Label defines a standard label information
-type Label struct {
- Name string
- Color string
- Description string
-}
+++ /dev/null
-// Copyright 2021 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-// Messenger is a formatting function similar to i18n.Tr
-type Messenger func(key string, args ...interface{})
-
-// NilMessenger represents an empty formatting function
-func NilMessenger(string, ...interface{}) {}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import "time"
-
-// Milestone defines a standard milestone
-type Milestone struct {
- Title string
- Description string
- Deadline *time.Time
- Created time.Time
- Updated *time.Time
- Closed *time.Time
- State string // open, closed
-}
+++ /dev/null
-// Copyright 2021 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import (
- "context"
- "net/url"
-)
-
-// NullDownloader implements a blank downloader
-type NullDownloader struct {
-}
-
-var (
- _ Downloader = &NullDownloader{}
-)
-
-// SetContext set context
-func (n NullDownloader) SetContext(_ context.Context) {}
-
-// GetRepoInfo returns a repository information
-func (n NullDownloader) GetRepoInfo() (*Repository, error) {
- return nil, &ErrNotSupported{Entity: "RepoInfo"}
-}
-
-// GetTopics return repository topics
-func (n NullDownloader) GetTopics() ([]string, error) {
- return nil, &ErrNotSupported{Entity: "Topics"}
-}
-
-// GetMilestones returns milestones
-func (n NullDownloader) GetMilestones() ([]*Milestone, error) {
- return nil, &ErrNotSupported{Entity: "Milestones"}
-}
-
-// GetReleases returns releases
-func (n NullDownloader) GetReleases() ([]*Release, error) {
- return nil, &ErrNotSupported{Entity: "Releases"}
-}
-
-// GetLabels returns labels
-func (n NullDownloader) GetLabels() ([]*Label, error) {
- return nil, &ErrNotSupported{Entity: "Labels"}
-}
-
-// GetIssues returns issues according start and limit
-func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
- return nil, false, &ErrNotSupported{Entity: "Issues"}
-}
-
-// GetComments returns comments according the options
-func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) {
- return nil, false, &ErrNotSupported{Entity: "Comments"}
-}
-
-// GetPullRequests returns pull requests according page and perPage
-func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
- return nil, false, &ErrNotSupported{Entity: "PullRequests"}
-}
-
-// GetReviews returns pull requests review
-func (n NullDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
- return nil, &ErrNotSupported{Entity: "Reviews"}
-}
-
-// FormatCloneURL add authentification into remote URLs
-func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
- if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
- u, err := url.Parse(remoteAddr)
- if err != nil {
- return "", err
- }
- u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
- if len(opts.AuthToken) > 0 {
- u.User = url.UserPassword("oauth2", opts.AuthToken)
- }
- return u.String(), nil
- }
- return remoteAddr, nil
-}
-
-// SupportGetRepoComments return true if it supports get repo comments
-func (n NullDownloader) SupportGetRepoComments() bool {
- return false
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import "code.gitea.io/gitea/modules/structs"
-
-// MigrateOptions defines the way a repository gets migrated
-// this is for internal usage by migrations module and func who interact with it
-type MigrateOptions struct {
- // required: true
- CloneAddr string `json:"clone_addr" binding:"Required"`
- CloneAddrEncrypted string `json:"clone_addr_encrypted,omitempty"`
- AuthUsername string `json:"auth_username"`
- AuthPassword string `json:"-"`
- AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"`
- AuthToken string `json:"-"`
- AuthTokenEncrypted string `json:"auth_token_encrypted,omitempty"`
- // required: true
- UID int `json:"uid" binding:"Required"`
- // required: true
- RepoName string `json:"repo_name" binding:"Required"`
- Mirror bool `json:"mirror"`
- LFS bool `json:"lfs"`
- LFSEndpoint string `json:"lfs_endpoint"`
- Private bool `json:"private"`
- Description string `json:"description"`
- OriginalURL string
- GitServiceType structs.GitServiceType
- Wiki bool
- Issues bool
- Milestones bool
- Labels bool
- Releases bool
- Comments bool
- PullRequests bool
- ReleaseAssets bool
- MigrateToRepoID int64
- MirrorInterval string `json:"mirror_interval"`
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import (
- "fmt"
- "time"
-)
-
-// PullRequest defines a standard pull request information
-type PullRequest struct {
- Number int64
- Title string
- PosterName string `yaml:"poster_name"`
- PosterID int64 `yaml:"poster_id"`
- PosterEmail string `yaml:"poster_email"`
- Content string
- Milestone string
- State string
- Created time.Time
- Updated time.Time
- Closed *time.Time
- Labels []*Label
- PatchURL string `yaml:"patch_url"`
- Merged bool
- MergedTime *time.Time `yaml:"merged_time"`
- MergeCommitSHA string `yaml:"merge_commit_sha"`
- Head PullRequestBranch
- Base PullRequestBranch
- Assignees []string
- IsLocked bool `yaml:"is_locked"`
- Reactions []*Reaction
- Context IssueContext `yaml:"-"`
-}
-
-// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
-func (p *PullRequest) IsForkPullRequest() bool {
- return p.Head.RepoPath() != p.Base.RepoPath()
-}
-
-// GetGitRefName returns pull request relative path to head
-func (p PullRequest) GetGitRefName() string {
- return fmt.Sprintf("refs/pull/%d/head", p.Number)
-}
-
-// PullRequestBranch represents a pull request branch
-type PullRequestBranch struct {
- CloneURL string `yaml:"clone_url"`
- Ref string
- SHA string
- RepoName string `yaml:"repo_name"`
- OwnerName string `yaml:"owner_name"`
-}
-
-// RepoPath returns pull request repo path
-func (p PullRequestBranch) RepoPath() string {
- return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName)
-}
+++ /dev/null
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-// Reaction represents a reaction to an issue/pr/comment.
-type Reaction struct {
- UserID int64 `yaml:"user_id"`
- UserName string `yaml:"user_name"`
- Content string
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import (
- "io"
- "time"
-)
-
-// ReleaseAsset represents a release asset
-type ReleaseAsset struct {
- ID int64
- Name string
- ContentType *string `yaml:"content_type"`
- Size *int
- DownloadCount *int `yaml:"download_count"`
- Created time.Time
- Updated time.Time
- DownloadURL *string `yaml:"download_url"`
- // if DownloadURL is nil, the function should be invoked
- DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
-}
-
-// Release represents a release
-type Release struct {
- TagName string `yaml:"tag_name"`
- TargetCommitish string `yaml:"target_commitish"`
- Name string
- Body string
- Draft bool
- Prerelease bool
- PublisherID int64 `yaml:"publisher_id"`
- PublisherName string `yaml:"publisher_name"`
- PublisherEmail string `yaml:"publisher_email"`
- Assets []*ReleaseAsset
- Created time.Time
- Published time.Time
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-// Repository defines a standard repository information
-type Repository struct {
- Name string
- Owner string
- IsPrivate bool `yaml:"is_private"`
- IsMirror bool `yaml:"is_mirror"`
- Description string
- CloneURL string `yaml:"clone_url"`
- OriginalURL string `yaml:"original_url"`
- DefaultBranch string
-}
+++ /dev/null
-// Copyright 2021 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import (
- "context"
- "time"
-)
-
-var (
- _ Downloader = &RetryDownloader{}
-)
-
-// RetryDownloader retry the downloads
-type RetryDownloader struct {
- Downloader
- ctx context.Context
- RetryTimes int // the total execute times
- RetryDelay int // time to delay seconds
-}
-
-// NewRetryDownloader creates a retry downloader
-func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
- return &RetryDownloader{
- Downloader: downloader,
- ctx: ctx,
- RetryTimes: retryTimes,
- RetryDelay: retryDelay,
- }
-}
-
-func (d *RetryDownloader) retry(work func() error) error {
- var (
- times = d.RetryTimes
- err error
- )
- for ; times > 0; times-- {
- if err = work(); err == nil {
- return nil
- }
- if IsErrNotSupported(err) {
- return err
- }
- select {
- case <-d.ctx.Done():
- return d.ctx.Err()
- case <-time.After(time.Second * time.Duration(d.RetryDelay)):
- }
- }
- return err
-}
-
-// SetContext set context
-func (d *RetryDownloader) SetContext(ctx context.Context) {
- d.ctx = ctx
- d.Downloader.SetContext(ctx)
-}
-
-// GetRepoInfo returns a repository information with retry
-func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
- var (
- repo *Repository
- err error
- )
-
- err = d.retry(func() error {
- repo, err = d.Downloader.GetRepoInfo()
- return err
- })
-
- return repo, err
-}
-
-// GetTopics returns a repository's topics with retry
-func (d *RetryDownloader) GetTopics() ([]string, error) {
- var (
- topics []string
- err error
- )
-
- err = d.retry(func() error {
- topics, err = d.Downloader.GetTopics()
- return err
- })
-
- return topics, err
-}
-
-// GetMilestones returns a repository's milestones with retry
-func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
- var (
- milestones []*Milestone
- err error
- )
-
- err = d.retry(func() error {
- milestones, err = d.Downloader.GetMilestones()
- return err
- })
-
- return milestones, err
-}
-
-// GetReleases returns a repository's releases with retry
-func (d *RetryDownloader) GetReleases() ([]*Release, error) {
- var (
- releases []*Release
- err error
- )
-
- err = d.retry(func() error {
- releases, err = d.Downloader.GetReleases()
- return err
- })
-
- return releases, err
-}
-
-// GetLabels returns a repository's labels with retry
-func (d *RetryDownloader) GetLabels() ([]*Label, error) {
- var (
- labels []*Label
- err error
- )
-
- err = d.retry(func() error {
- labels, err = d.Downloader.GetLabels()
- return err
- })
-
- return labels, err
-}
-
-// GetIssues returns a repository's issues with retry
-func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
- var (
- issues []*Issue
- isEnd bool
- err error
- )
-
- err = d.retry(func() error {
- issues, isEnd, err = d.Downloader.GetIssues(page, perPage)
- return err
- })
-
- return issues, isEnd, err
-}
-
-// GetComments returns a repository's comments with retry
-func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) {
- var (
- comments []*Comment
- isEnd bool
- err error
- )
-
- err = d.retry(func() error {
- comments, isEnd, err = d.Downloader.GetComments(opts)
- return err
- })
-
- return comments, isEnd, err
-}
-
-// GetPullRequests returns a repository's pull requests with retry
-func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
- var (
- prs []*PullRequest
- err error
- isEnd bool
- )
-
- err = d.retry(func() error {
- prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage)
- return err
- })
-
- return prs, isEnd, err
-}
-
-// GetReviews returns pull requests reviews
-func (d *RetryDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
- var (
- reviews []*Review
- err error
- )
-
- err = d.retry(func() error {
- reviews, err = d.Downloader.GetReviews(pullRequestContext)
- return err
- })
-
- return reviews, err
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-import "time"
-
-// enumerate all review states
-const (
- ReviewStatePending = "PENDING"
- ReviewStateApproved = "APPROVED"
- ReviewStateChangesRequested = "CHANGES_REQUESTED"
- ReviewStateCommented = "COMMENTED"
-)
-
-// Review is a standard review information
-type Review struct {
- ID int64
- IssueIndex int64 `yaml:"issue_index"`
- ReviewerID int64 `yaml:"reviewer_id"`
- ReviewerName string `yaml:"reviewer_name"`
- Official bool
- CommitID string `yaml:"commit_id"`
- Content string
- CreatedAt time.Time `yaml:"created_at"`
- State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
- Comments []*ReviewComment
-}
-
-// ReviewComment represents a review comment
-type ReviewComment struct {
- ID int64
- InReplyTo int64 `yaml:"in_reply_to"`
- Content string
- TreePath string `yaml:"tree_path"`
- DiffHunk string `yaml:"diff_hunk"`
- Position int
- Line int
- CommitID string `yaml:"commit_id"`
- PosterID int64 `yaml:"poster_id"`
- Reactions []*Reaction
- CreatedAt time.Time `yaml:"created_at"`
- UpdatedAt time.Time `yaml:"updated_at"`
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package base
-
-// Uploader uploads all the information of one repository
-type Uploader interface {
- MaxBatchInsertSize(tp string) int
- CreateRepo(repo *Repository, opts MigrateOptions) error
- CreateTopics(topic ...string) error
- CreateMilestones(milestones ...*Milestone) error
- CreateReleases(releases ...*Release) error
- SyncTags() error
- CreateLabels(labels ...*Label) error
- CreateIssues(issues ...*Issue) error
- CreateComments(comments ...*Comment) error
- CreatePullRequests(prs ...*PullRequest) error
- CreateReviews(reviews ...*Review) error
- Rollback() error
- Finish() error
- Close()
-}
+++ /dev/null
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "os"
- "path"
- "path/filepath"
- "strconv"
- "strings"
- "time"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/git"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/repository"
- "code.gitea.io/gitea/modules/structs"
-
- "gopkg.in/yaml.v2"
-)
-
-var (
- _ base.Uploader = &RepositoryDumper{}
-)
-
-// RepositoryDumper implements an Uploader to the local directory
-type RepositoryDumper struct {
- ctx context.Context
- baseDir string
- repoOwner string
- repoName string
- opts base.MigrateOptions
- milestoneFile *os.File
- labelFile *os.File
- releaseFile *os.File
- issueFile *os.File
- commentFiles map[int64]*os.File
- pullrequestFile *os.File
- reviewFiles map[int64]*os.File
-
- gitRepo *git.Repository
- prHeadCache map[string]struct{}
-}
-
-// NewRepositoryDumper creates an gitea Uploader
-func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) (*RepositoryDumper, error) {
- baseDir = filepath.Join(baseDir, repoOwner, repoName)
- if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
- return nil, err
- }
- return &RepositoryDumper{
- ctx: ctx,
- opts: opts,
- baseDir: baseDir,
- repoOwner: repoOwner,
- repoName: repoName,
- prHeadCache: make(map[string]struct{}),
- commentFiles: make(map[int64]*os.File),
- reviewFiles: make(map[int64]*os.File),
- }, nil
-}
-
-// MaxBatchInsertSize returns the table's max batch insert size
-func (g *RepositoryDumper) MaxBatchInsertSize(tp string) int {
- return 1000
-}
-
-func (g *RepositoryDumper) gitPath() string {
- return filepath.Join(g.baseDir, "git")
-}
-
-func (g *RepositoryDumper) wikiPath() string {
- return filepath.Join(g.baseDir, "wiki")
-}
-
-func (g *RepositoryDumper) commentDir() string {
- return filepath.Join(g.baseDir, "comments")
-}
-
-func (g *RepositoryDumper) reviewDir() string {
- return filepath.Join(g.baseDir, "reviews")
-}
-
-func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
- if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 {
- u, err := url.Parse(remoteAddr)
- if err != nil {
- return "", err
- }
- u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword)
- if len(g.opts.AuthToken) > 0 {
- u.User = url.UserPassword("oauth2", g.opts.AuthToken)
- }
- remoteAddr = u.String()
- }
-
- return remoteAddr, nil
-}
-
-// CreateRepo creates a repository
-func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
- f, err := os.Create(filepath.Join(g.baseDir, "repo.yml"))
- if err != nil {
- return err
- }
- defer f.Close()
-
- bs, err := yaml.Marshal(map[string]interface{}{
- "name": repo.Name,
- "owner": repo.Owner,
- "description": repo.Description,
- "clone_addr": opts.CloneAddr,
- "original_url": repo.OriginalURL,
- "is_private": opts.Private,
- "service_type": opts.GitServiceType,
- "wiki": opts.Wiki,
- "issues": opts.Issues,
- "milestones": opts.Milestones,
- "labels": opts.Labels,
- "releases": opts.Releases,
- "comments": opts.Comments,
- "pulls": opts.PullRequests,
- "assets": opts.ReleaseAssets,
- })
- if err != nil {
- return err
- }
-
- if _, err := f.Write(bs); err != nil {
- return err
- }
-
- repoPath := g.gitPath()
- if err := os.MkdirAll(repoPath, os.ModePerm); err != nil {
- return err
- }
-
- migrateTimeout := 2 * time.Hour
-
- remoteAddr, err := g.setURLToken(repo.CloneURL)
- if err != nil {
- return err
- }
-
- err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{
- Mirror: true,
- Quiet: true,
- Timeout: migrateTimeout,
- })
- if err != nil {
- return fmt.Errorf("Clone: %v", err)
- }
-
- if opts.Wiki {
- wikiPath := g.wikiPath()
- wikiRemotePath := repository.WikiRemoteURL(remoteAddr)
- if len(wikiRemotePath) > 0 {
- if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil {
- return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
- }
-
- if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
- Mirror: true,
- Quiet: true,
- Timeout: migrateTimeout,
- Branch: "master",
- }); err != nil {
- log.Warn("Clone wiki: %v", err)
- if err := os.RemoveAll(wikiPath); err != nil {
- return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
- }
- }
- }
- }
-
- g.gitRepo, err = git.OpenRepository(g.gitPath())
- return err
-}
-
-// Close closes this uploader
-func (g *RepositoryDumper) Close() {
- if g.gitRepo != nil {
- g.gitRepo.Close()
- }
- if g.milestoneFile != nil {
- g.milestoneFile.Close()
- }
- if g.labelFile != nil {
- g.labelFile.Close()
- }
- if g.releaseFile != nil {
- g.releaseFile.Close()
- }
- if g.issueFile != nil {
- g.issueFile.Close()
- }
- for _, f := range g.commentFiles {
- f.Close()
- }
- if g.pullrequestFile != nil {
- g.pullrequestFile.Close()
- }
- for _, f := range g.reviewFiles {
- f.Close()
- }
-}
-
-// CreateTopics creates topics
-func (g *RepositoryDumper) CreateTopics(topics ...string) error {
- f, err := os.Create(filepath.Join(g.baseDir, "topic.yml"))
- if err != nil {
- return err
- }
- defer f.Close()
-
- bs, err := yaml.Marshal(map[string]interface{}{
- "topics": topics,
- })
- if err != nil {
- return err
- }
-
- if _, err := f.Write(bs); err != nil {
- return err
- }
-
- return nil
-}
-
-// CreateMilestones creates milestones
-func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error {
- var err error
- if g.milestoneFile == nil {
- g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml"))
- if err != nil {
- return err
- }
- }
-
- bs, err := yaml.Marshal(milestones)
- if err != nil {
- return err
- }
-
- if _, err := g.milestoneFile.Write(bs); err != nil {
- return err
- }
-
- return nil
-}
-
-// CreateLabels creates labels
-func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error {
- var err error
- if g.labelFile == nil {
- g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml"))
- if err != nil {
- return err
- }
- }
-
- bs, err := yaml.Marshal(labels)
- if err != nil {
- return err
- }
-
- if _, err := g.labelFile.Write(bs); err != nil {
- return err
- }
-
- return nil
-}
-
-// CreateReleases creates releases
-func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error {
- if g.opts.ReleaseAssets {
- for _, release := range releases {
- attachDir := filepath.Join("release_assets", release.TagName)
- if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil {
- return err
- }
- for _, asset := range release.Assets {
- attachLocalPath := filepath.Join(attachDir, asset.Name)
- // download attachment
-
- err := func(attachPath string) error {
- var rc io.ReadCloser
- var err error
- if asset.DownloadURL == nil {
- rc, err = asset.DownloadFunc()
- if err != nil {
- return err
- }
- } else {
- resp, err := http.Get(*asset.DownloadURL)
- if err != nil {
- return err
- }
- rc = resp.Body
- }
- defer rc.Close()
-
- fw, err := os.Create(attachPath)
- if err != nil {
- return fmt.Errorf("Create: %v", err)
- }
- defer fw.Close()
-
- _, err = io.Copy(fw, rc)
- return err
- }(filepath.Join(g.baseDir, attachLocalPath))
- if err != nil {
- return err
- }
- asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source
- }
- }
- }
-
- var err error
- if g.releaseFile == nil {
- g.releaseFile, err = os.Create(filepath.Join(g.baseDir, "release.yml"))
- if err != nil {
- return err
- }
- }
-
- bs, err := yaml.Marshal(releases)
- if err != nil {
- return err
- }
-
- if _, err := g.releaseFile.Write(bs); err != nil {
- return err
- }
-
- return nil
-}
-
-// SyncTags syncs releases with tags in the database
-func (g *RepositoryDumper) SyncTags() error {
- return nil
-}
-
-// CreateIssues creates issues
-func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error {
- var err error
- if g.issueFile == nil {
- g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml"))
- if err != nil {
- return err
- }
- }
-
- bs, err := yaml.Marshal(issues)
- if err != nil {
- return err
- }
-
- if _, err := g.issueFile.Write(bs); err != nil {
- return err
- }
-
- return nil
-}
-
-func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File, itemsMap map[int64][]interface{}) error {
- if err := os.MkdirAll(dir, os.ModePerm); err != nil {
- return err
- }
-
- for number, items := range itemsMap {
- var err error
- itemFile := itemFiles[number]
- if itemFile == nil {
- itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number)))
- if err != nil {
- return err
- }
- itemFiles[number] = itemFile
- }
-
- bs, err := yaml.Marshal(items)
- if err != nil {
- return err
- }
-
- if _, err := itemFile.Write(bs); err != nil {
- return err
- }
- }
-
- return nil
-}
-
-// CreateComments creates comments of issues
-func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error {
- var commentsMap = make(map[int64][]interface{}, len(comments))
- for _, comment := range comments {
- commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment)
- }
-
- return g.createItems(g.commentDir(), g.commentFiles, commentsMap)
-}
-
-// CreatePullRequests creates pull requests
-func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
- for _, pr := range prs {
- // download patch file
- err := func() error {
- u, err := g.setURLToken(pr.PatchURL)
- if err != nil {
- return err
- }
- resp, err := http.Get(u)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
- pullDir := filepath.Join(g.gitPath(), "pulls")
- if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
- return err
- }
- fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))
- f, err := os.Create(fPath)
- if err != nil {
- return err
- }
- defer f.Close()
- if _, err = io.Copy(f, resp.Body); err != nil {
- return err
- }
- pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number)
-
- return nil
- }()
- if err != nil {
- return err
- }
-
- // set head information
- pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
- if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
- return err
- }
- p, err := os.Create(filepath.Join(pullHead, "head"))
- if err != nil {
- return err
- }
- _, err = p.WriteString(pr.Head.SHA)
- p.Close()
- if err != nil {
- return err
- }
-
- if pr.IsForkPullRequest() && pr.State != "closed" {
- if pr.Head.OwnerName != "" {
- remote := pr.Head.OwnerName
- _, ok := g.prHeadCache[remote]
- if !ok {
- // git remote add
- // TODO: how to handle private CloneURL?
- err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
- if err != nil {
- log.Error("AddRemote failed: %s", err)
- } else {
- g.prHeadCache[remote] = struct{}{}
- ok = true
- }
- }
-
- if ok {
- _, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.gitPath())
- if err != nil {
- log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
- } else {
- // a new branch name with <original_owner_name/original_branchname> will be created to as new head branch
- ref := path.Join(pr.Head.OwnerName, pr.Head.Ref)
- headBranch := filepath.Join(g.gitPath(), "refs", "heads", ref)
- if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
- return err
- }
- b, err := os.Create(headBranch)
- if err != nil {
- return err
- }
- _, err = b.WriteString(pr.Head.SHA)
- b.Close()
- if err != nil {
- return err
- }
- pr.Head.Ref = ref
- }
- }
- }
- }
- // whatever it's a forked repo PR, we have to change head info as the same as the base info
- pr.Head.OwnerName = pr.Base.OwnerName
- pr.Head.RepoName = pr.Base.RepoName
- }
-
- var err error
- if g.pullrequestFile == nil {
- if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil {
- return err
- }
- g.pullrequestFile, err = os.Create(filepath.Join(g.baseDir, "pull_request.yml"))
- if err != nil {
- return err
- }
- }
-
- bs, err := yaml.Marshal(prs)
- if err != nil {
- return err
- }
-
- if _, err := g.pullrequestFile.Write(bs); err != nil {
- return err
- }
-
- return nil
-}
-
-// CreateReviews create pull request reviews
-func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error {
- var reviewsMap = make(map[int64][]interface{}, len(reviews))
- for _, review := range reviews {
- reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review)
- }
-
- return g.createItems(g.reviewDir(), g.reviewFiles, reviewsMap)
-}
-
-// Rollback when migrating failed, this will rollback all the changes.
-func (g *RepositoryDumper) Rollback() error {
- g.Close()
- return os.RemoveAll(g.baseDir)
-}
-
-// Finish when migrating succeed, this will update something.
-func (g *RepositoryDumper) Finish() error {
- return nil
-}
-
-// DumpRepository dump repository according MigrateOptions to a local directory
-func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error {
- downloader, err := newDownloader(ctx, ownerName, opts)
- if err != nil {
- return err
- }
- uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts)
- if err != nil {
- return err
- }
-
- if err := migrateRepository(downloader, uploader, opts, nil); err != nil {
- if err1 := uploader.Rollback(); err1 != nil {
- log.Error("rollback failed: %v", err1)
- }
- return err
- }
- return nil
-}
-
-func updateOptionsUnits(opts *base.MigrateOptions, units []string) {
- if len(units) == 0 {
- opts.Wiki = true
- opts.Issues = true
- opts.Milestones = true
- opts.Labels = true
- opts.Releases = true
- opts.Comments = true
- opts.PullRequests = true
- opts.ReleaseAssets = true
- } else {
- for _, unit := range units {
- switch strings.ToLower(unit) {
- case "wiki":
- opts.Wiki = true
- case "issues":
- opts.Issues = true
- case "milestones":
- opts.Milestones = true
- case "labels":
- opts.Labels = true
- case "releases":
- opts.Releases = true
- case "release_assets":
- opts.ReleaseAssets = true
- case "comments":
- opts.Comments = true
- case "pull_requests":
- opts.PullRequests = true
- }
- }
- }
-}
-
-// RestoreRepository restore a repository from the disk directory
-func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string, units []string) error {
- doer, err := models.GetAdminUser()
- if err != nil {
- return err
- }
- var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName)
- downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName)
- if err != nil {
- return err
- }
- opts, err := downloader.getRepoOptions()
- if err != nil {
- return err
- }
- tp, _ := strconv.Atoi(opts["service_type"])
-
- var migrateOpts = base.MigrateOptions{
- GitServiceType: structs.GitServiceType(tp),
- }
- updateOptionsUnits(&migrateOpts, units)
-
- if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil {
- if err1 := uploader.Rollback(); err1 != nil {
- log.Error("rollback failed: %v", err1)
- }
- return err
- }
- return updateMigrationPosterIDByGitService(ctx, structs.GitServiceType(tp))
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "errors"
-
- "github.com/google/go-github/v39/github"
-)
-
-var (
- // ErrRepoNotCreated returns the error that repository not created
- ErrRepoNotCreated = errors.New("repository is not created yet")
-)
-
-// IsRateLimitError returns true if the err is github.RateLimitError
-func IsRateLimitError(err error) bool {
- _, ok := err.(*github.RateLimitError)
- return ok
-}
-
-// IsTwoFactorAuthError returns true if the err is github.TwoFactorAuthError
-func IsTwoFactorAuthError(err error) bool {
- _, ok := err.(*github.TwoFactorAuthError)
- return ok
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
-
- "code.gitea.io/gitea/modules/migrations/base"
-)
-
-var (
- _ base.Downloader = &PlainGitDownloader{}
-)
-
-// PlainGitDownloader implements a Downloader interface to clone git from a http/https URL
-type PlainGitDownloader struct {
- base.NullDownloader
- ownerName string
- repoName string
- remoteURL string
-}
-
-// NewPlainGitDownloader creates a git Downloader
-func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownloader {
- return &PlainGitDownloader{
- ownerName: ownerName,
- repoName: repoName,
- remoteURL: remoteURL,
- }
-}
-
-// SetContext set context
-func (g *PlainGitDownloader) SetContext(ctx context.Context) {
-}
-
-// GetRepoInfo returns a repository information
-func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
- // convert github repo to stand Repo
- return &base.Repository{
- Owner: g.ownerName,
- Name: g.repoName,
- CloneURL: g.remoteURL,
- }, nil
-}
-
-// GetTopics return empty string slice
-func (g PlainGitDownloader) GetTopics() ([]string, error) {
- return []string{}, nil
-}
+++ /dev/null
-// Copyright 2021 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "net/url"
- "strings"
-
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/structs"
-)
-
-var (
- _ base.Downloader = &GitBucketDownloader{}
- _ base.DownloaderFactory = &GitBucketDownloaderFactory{}
-)
-
-func init() {
- RegisterDownloaderFactory(&GitBucketDownloaderFactory{})
-}
-
-// GitBucketDownloaderFactory defines a GitBucket downloader factory
-type GitBucketDownloaderFactory struct {
-}
-
-// New returns a Downloader related to this factory according MigrateOptions
-func (f *GitBucketDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
- u, err := url.Parse(opts.CloneAddr)
- if err != nil {
- return nil, err
- }
-
- baseURL := u.Scheme + "://" + u.Host
- fields := strings.Split(u.Path, "/")
- oldOwner := fields[1]
- oldName := strings.TrimSuffix(fields[2], ".git")
-
- return NewGitBucketDownloader(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
-}
-
-// GitServiceType returns the type of git service
-func (f *GitBucketDownloaderFactory) GitServiceType() structs.GitServiceType {
- return structs.GitBucketService
-}
-
-// GitBucketDownloader implements a Downloader interface to get repository information
-// from GitBucket via GithubDownloader
-type GitBucketDownloader struct {
- *GithubDownloaderV3
-}
-
-// NewGitBucketDownloader creates a GitBucket downloader
-func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GitBucketDownloader {
- githubDownloader := NewGithubDownloaderV3(ctx, baseURL, userName, password, token, repoOwner, repoName)
- githubDownloader.SkipReactions = true
- return &GitBucketDownloader{
- githubDownloader,
- }
-}
-
-// SupportGetRepoComments return true if it supports get repo comments
-func (g *GitBucketDownloader) SupportGetRepoComments() bool {
- return false
-}
-
-// GetReviews is not supported
-func (g *GitBucketDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
- return nil, &base.ErrNotSupported{Entity: "Reviews"}
-}
+++ /dev/null
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "crypto/tls"
- "errors"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "strings"
- "time"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/proxy"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/structs"
-
- gitea_sdk "code.gitea.io/sdk/gitea"
-)
-
-var (
- _ base.Downloader = &GiteaDownloader{}
- _ base.DownloaderFactory = &GiteaDownloaderFactory{}
-)
-
-func init() {
- RegisterDownloaderFactory(&GiteaDownloaderFactory{})
-}
-
-// GiteaDownloaderFactory defines a gitea downloader factory
-type GiteaDownloaderFactory struct {
-}
-
-// New returns a Downloader related to this factory according MigrateOptions
-func (f *GiteaDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
- u, err := url.Parse(opts.CloneAddr)
- if err != nil {
- return nil, err
- }
-
- baseURL := u.Scheme + "://" + u.Host
- repoNameSpace := strings.TrimPrefix(u.Path, "/")
- repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
-
- path := strings.Split(repoNameSpace, "/")
- if len(path) < 2 {
- return nil, fmt.Errorf("invalid path: %s", repoNameSpace)
- }
-
- repoPath := strings.Join(path[len(path)-2:], "/")
- if len(path) > 2 {
- subPath := strings.Join(path[:len(path)-2], "/")
- baseURL += "/" + subPath
- }
-
- log.Trace("Create gitea downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
-
- return NewGiteaDownloader(ctx, baseURL, repoPath, opts.AuthUsername, opts.AuthPassword, opts.AuthToken)
-}
-
-// GitServiceType returns the type of git service
-func (f *GiteaDownloaderFactory) GitServiceType() structs.GitServiceType {
- return structs.GiteaService
-}
-
-// GiteaDownloader implements a Downloader interface to get repository information's
-type GiteaDownloader struct {
- base.NullDownloader
- ctx context.Context
- client *gitea_sdk.Client
- repoOwner string
- repoName string
- pagination bool
- maxPerPage int
-}
-
-// NewGiteaDownloader creates a gitea Downloader via gitea API
-// Use either a username/password or personal token. token is preferred
-// Note: Public access only allows very basic access
-func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GiteaDownloader, error) {
- giteaClient, err := gitea_sdk.NewClient(
- baseURL,
- gitea_sdk.SetToken(token),
- gitea_sdk.SetBasicAuth(username, password),
- gitea_sdk.SetContext(ctx),
- gitea_sdk.SetHTTPClient(&http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
- Proxy: proxy.Proxy(),
- },
- }),
- )
- if err != nil {
- log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
- return nil, err
- }
-
- path := strings.Split(repoPath, "/")
-
- paginationSupport := true
- if err = giteaClient.CheckServerVersionConstraint(">=1.12"); err != nil {
- paginationSupport = false
- }
-
- // set small maxPerPage since we can only guess
- // (default would be 50 but this can differ)
- maxPerPage := 10
- // gitea instances >=1.13 can tell us what maximum they have
- apiConf, _, err := giteaClient.GetGlobalAPISettings()
- if err != nil {
- log.Info("Unable to get global API settings. Ignoring these.")
- log.Debug("giteaClient.GetGlobalAPISettings. Error: %v", err)
- }
- if apiConf != nil {
- maxPerPage = apiConf.MaxResponseItems
- }
-
- return &GiteaDownloader{
- ctx: ctx,
- client: giteaClient,
- repoOwner: path[0],
- repoName: path[1],
- pagination: paginationSupport,
- maxPerPage: maxPerPage,
- }, nil
-}
-
-// SetContext set context
-func (g *GiteaDownloader) SetContext(ctx context.Context) {
- g.ctx = ctx
-}
-
-// GetRepoInfo returns a repository information
-func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) {
- if g == nil {
- return nil, errors.New("error: GiteaDownloader is nil")
- }
-
- repo, _, err := g.client.GetRepo(g.repoOwner, g.repoName)
- if err != nil {
- return nil, err
- }
-
- return &base.Repository{
- Name: repo.Name,
- Owner: repo.Owner.UserName,
- IsPrivate: repo.Private,
- Description: repo.Description,
- CloneURL: repo.CloneURL,
- OriginalURL: repo.HTMLURL,
- DefaultBranch: repo.DefaultBranch,
- }, nil
-}
-
-// GetTopics return gitea topics
-func (g *GiteaDownloader) GetTopics() ([]string, error) {
- topics, _, err := g.client.ListRepoTopics(g.repoOwner, g.repoName, gitea_sdk.ListRepoTopicsOptions{})
- return topics, err
-}
-
-// GetMilestones returns milestones
-func (g *GiteaDownloader) GetMilestones() ([]*base.Milestone, error) {
- var milestones = make([]*base.Milestone, 0, g.maxPerPage)
-
- for i := 1; ; i++ {
- // make sure gitea can shutdown gracefully
- select {
- case <-g.ctx.Done():
- return nil, nil
- default:
- }
-
- ms, _, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName, gitea_sdk.ListMilestoneOption{
- ListOptions: gitea_sdk.ListOptions{
- PageSize: g.maxPerPage,
- Page: i,
- },
- State: gitea_sdk.StateAll,
- })
- if err != nil {
- return nil, err
- }
-
- for i := range ms {
- // old gitea instances dont have this information
- createdAT := time.Time{}
- var updatedAT *time.Time
- if ms[i].Closed != nil {
- createdAT = *ms[i].Closed
- updatedAT = ms[i].Closed
- }
-
- // new gitea instances (>=1.13) do
- if !ms[i].Created.IsZero() {
- createdAT = ms[i].Created
- }
- if ms[i].Updated != nil && !ms[i].Updated.IsZero() {
- updatedAT = ms[i].Updated
- }
-
- milestones = append(milestones, &base.Milestone{
- Title: ms[i].Title,
- Description: ms[i].Description,
- Deadline: ms[i].Deadline,
- Created: createdAT,
- Updated: updatedAT,
- Closed: ms[i].Closed,
- State: string(ms[i].State),
- })
- }
- if !g.pagination || len(ms) < g.maxPerPage {
- break
- }
- }
- return milestones, nil
-}
-
-func (g *GiteaDownloader) convertGiteaLabel(label *gitea_sdk.Label) *base.Label {
- return &base.Label{
- Name: label.Name,
- Color: label.Color,
- Description: label.Description,
- }
-}
-
-// GetLabels returns labels
-func (g *GiteaDownloader) GetLabels() ([]*base.Label, error) {
- var labels = make([]*base.Label, 0, g.maxPerPage)
-
- for i := 1; ; i++ {
- // make sure gitea can shutdown gracefully
- select {
- case <-g.ctx.Done():
- return nil, nil
- default:
- }
-
- ls, _, err := g.client.ListRepoLabels(g.repoOwner, g.repoName, gitea_sdk.ListLabelsOptions{ListOptions: gitea_sdk.ListOptions{
- PageSize: g.maxPerPage,
- Page: i,
- }})
- if err != nil {
- return nil, err
- }
-
- for i := range ls {
- labels = append(labels, g.convertGiteaLabel(ls[i]))
- }
- if !g.pagination || len(ls) < g.maxPerPage {
- break
- }
- }
- return labels, nil
-}
-
-func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Release {
- r := &base.Release{
- TagName: rel.TagName,
- TargetCommitish: rel.Target,
- Name: rel.Title,
- Body: rel.Note,
- Draft: rel.IsDraft,
- Prerelease: rel.IsPrerelease,
- PublisherID: rel.Publisher.ID,
- PublisherName: rel.Publisher.UserName,
- PublisherEmail: rel.Publisher.Email,
- Published: rel.PublishedAt,
- Created: rel.CreatedAt,
- }
-
- httpClient := &http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
- Proxy: proxy.Proxy(),
- },
- }
-
- for _, asset := range rel.Attachments {
- size := int(asset.Size)
- dlCount := int(asset.DownloadCount)
- r.Assets = append(r.Assets, &base.ReleaseAsset{
- ID: asset.ID,
- Name: asset.Name,
- Size: &size,
- DownloadCount: &dlCount,
- Created: asset.Created,
- DownloadURL: &asset.DownloadURL,
- DownloadFunc: func() (io.ReadCloser, error) {
- asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, rel.ID, asset.ID)
- if err != nil {
- return nil, err
- }
- // FIXME: for a private download?
- req, err := http.NewRequest("GET", asset.DownloadURL, nil)
- if err != nil {
- return nil, err
- }
- resp, err := httpClient.Do(req)
- if err != nil {
- return nil, err
- }
-
- // resp.Body is closed by the uploader
- return resp.Body, nil
- },
- })
- }
- return r
-}
-
-// GetReleases returns releases
-func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) {
- var releases = make([]*base.Release, 0, g.maxPerPage)
-
- for i := 1; ; i++ {
- // make sure gitea can shutdown gracefully
- select {
- case <-g.ctx.Done():
- return nil, nil
- default:
- }
-
- rl, _, err := g.client.ListReleases(g.repoOwner, g.repoName, gitea_sdk.ListReleasesOptions{ListOptions: gitea_sdk.ListOptions{
- PageSize: g.maxPerPage,
- Page: i,
- }})
- if err != nil {
- return nil, err
- }
-
- for i := range rl {
- releases = append(releases, g.convertGiteaRelease(rl[i]))
- }
- if !g.pagination || len(rl) < g.maxPerPage {
- break
- }
- }
- return releases, nil
-}
-
-func (g *GiteaDownloader) getIssueReactions(index int64) ([]*base.Reaction, error) {
- var reactions []*base.Reaction
- if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
- log.Info("GiteaDownloader: instance to old, skip getIssueReactions")
- return reactions, nil
- }
- rl, _, err := g.client.GetIssueReactions(g.repoOwner, g.repoName, index)
- if err != nil {
- return nil, err
- }
-
- for _, reaction := range rl {
- reactions = append(reactions, &base.Reaction{
- UserID: reaction.User.ID,
- UserName: reaction.User.UserName,
- Content: reaction.Reaction,
- })
- }
- return reactions, nil
-}
-
-func (g *GiteaDownloader) getCommentReactions(commentID int64) ([]*base.Reaction, error) {
- var reactions []*base.Reaction
- if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
- log.Info("GiteaDownloader: instance to old, skip getCommentReactions")
- return reactions, nil
- }
- rl, _, err := g.client.GetIssueCommentReactions(g.repoOwner, g.repoName, commentID)
- if err != nil {
- return nil, err
- }
-
- for i := range rl {
- reactions = append(reactions, &base.Reaction{
- UserID: rl[i].User.ID,
- UserName: rl[i].User.UserName,
- Content: rl[i].Reaction,
- })
- }
- return reactions, nil
-}
-
-// GetIssues returns issues according start and limit
-func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
- var allIssues = make([]*base.Issue, 0, perPage)
-
- issues, _, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gitea_sdk.ListIssueOption{
- ListOptions: gitea_sdk.ListOptions{Page: page, PageSize: perPage},
- State: gitea_sdk.StateAll,
- Type: gitea_sdk.IssueTypeIssue,
- })
- if err != nil {
- return nil, false, fmt.Errorf("error while listing issues: %v", err)
- }
- for _, issue := range issues {
-
- var labels = make([]*base.Label, 0, len(issue.Labels))
- for i := range issue.Labels {
- labels = append(labels, g.convertGiteaLabel(issue.Labels[i]))
- }
-
- var milestone string
- if issue.Milestone != nil {
- milestone = issue.Milestone.Title
- }
-
- reactions, err := g.getIssueReactions(issue.Index)
- if err != nil {
- log.Warn("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)
- if err2 := models.CreateRepositoryNotice(
- fmt.Sprintf("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)); err2 != nil {
- log.Error("create repository notice failed: ", err2)
- }
- }
-
- var assignees []string
- for i := range issue.Assignees {
- assignees = append(assignees, issue.Assignees[i].UserName)
- }
-
- allIssues = append(allIssues, &base.Issue{
- Title: issue.Title,
- Number: issue.Index,
- PosterID: issue.Poster.ID,
- PosterName: issue.Poster.UserName,
- PosterEmail: issue.Poster.Email,
- Content: issue.Body,
- Milestone: milestone,
- State: string(issue.State),
- Created: issue.Created,
- Updated: issue.Updated,
- Closed: issue.Closed,
- Reactions: reactions,
- Labels: labels,
- Assignees: assignees,
- IsLocked: issue.IsLocked,
- Context: base.BasicIssueContext(issue.Index),
- })
- }
-
- isEnd := len(issues) < perPage
- if !g.pagination {
- isEnd = len(issues) == 0
- }
- return allIssues, isEnd, nil
-}
-
-// GetComments returns comments according issueNumber
-func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
- var allComments = make([]*base.Comment, 0, g.maxPerPage)
-
- for i := 1; ; i++ {
- // make sure gitea can shutdown gracefully
- select {
- case <-g.ctx.Done():
- return nil, false, nil
- default:
- }
-
- comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{
- PageSize: g.maxPerPage,
- Page: i,
- }})
- if err != nil {
- return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.Context.ForeignID(), err)
- }
-
- for _, comment := range comments {
- reactions, err := g.getCommentReactions(comment.ID)
- if err != nil {
- log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)
- if err2 := models.CreateRepositoryNotice(
- fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
- log.Error("create repository notice failed: ", err2)
- }
- }
-
- allComments = append(allComments, &base.Comment{
- IssueIndex: opts.Context.LocalID(),
- PosterID: comment.Poster.ID,
- PosterName: comment.Poster.UserName,
- PosterEmail: comment.Poster.Email,
- Content: comment.Body,
- Created: comment.Created,
- Updated: comment.Updated,
- Reactions: reactions,
- })
- }
-
- if !g.pagination || len(comments) < g.maxPerPage {
- break
- }
- }
- return allComments, true, nil
-}
-
-// GetPullRequests returns pull requests according page and perPage
-func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
- var allPRs = make([]*base.PullRequest, 0, perPage)
-
- prs, _, err := g.client.ListRepoPullRequests(g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
- ListOptions: gitea_sdk.ListOptions{
- Page: page,
- PageSize: perPage,
- },
- State: gitea_sdk.StateAll,
- })
- if err != nil {
- return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %v", page, perPage, err)
- }
- for _, pr := range prs {
- var milestone string
- if pr.Milestone != nil {
- milestone = pr.Milestone.Title
- }
-
- var labels = make([]*base.Label, 0, len(pr.Labels))
- for i := range pr.Labels {
- labels = append(labels, g.convertGiteaLabel(pr.Labels[i]))
- }
-
- var (
- headUserName string
- headRepoName string
- headCloneURL string
- headRef string
- headSHA string
- )
- if pr.Head != nil {
- if pr.Head.Repository != nil {
- headUserName = pr.Head.Repository.Owner.UserName
- headRepoName = pr.Head.Repository.Name
- headCloneURL = pr.Head.Repository.CloneURL
- }
- headSHA = pr.Head.Sha
- headRef = pr.Head.Ref
- }
-
- var mergeCommitSHA string
- if pr.MergedCommitID != nil {
- mergeCommitSHA = *pr.MergedCommitID
- }
-
- reactions, err := g.getIssueReactions(pr.Index)
- if err != nil {
- log.Warn("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)
- if err2 := models.CreateRepositoryNotice(
- fmt.Sprintf("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)); err2 != nil {
- log.Error("create repository notice failed: ", err2)
- }
- }
-
- var assignees []string
- for i := range pr.Assignees {
- assignees = append(assignees, pr.Assignees[i].UserName)
- }
-
- createdAt := time.Time{}
- if pr.Created != nil {
- createdAt = *pr.Created
- }
- updatedAt := time.Time{}
- if pr.Created != nil {
- updatedAt = *pr.Updated
- }
-
- closedAt := pr.Closed
- if pr.Merged != nil && closedAt == nil {
- closedAt = pr.Merged
- }
-
- allPRs = append(allPRs, &base.PullRequest{
- Title: pr.Title,
- Number: pr.Index,
- PosterID: pr.Poster.ID,
- PosterName: pr.Poster.UserName,
- PosterEmail: pr.Poster.Email,
- Content: pr.Body,
- State: string(pr.State),
- Created: createdAt,
- Updated: updatedAt,
- Closed: closedAt,
- Labels: labels,
- Milestone: milestone,
- Reactions: reactions,
- Assignees: assignees,
- Merged: pr.HasMerged,
- MergedTime: pr.Merged,
- MergeCommitSHA: mergeCommitSHA,
- IsLocked: pr.IsLocked,
- PatchURL: pr.PatchURL,
- Head: base.PullRequestBranch{
- Ref: headRef,
- SHA: headSHA,
- RepoName: headRepoName,
- OwnerName: headUserName,
- CloneURL: headCloneURL,
- },
- Base: base.PullRequestBranch{
- Ref: pr.Base.Ref,
- SHA: pr.Base.Sha,
- RepoName: g.repoName,
- OwnerName: g.repoOwner,
- },
- Context: base.BasicIssueContext(pr.Index),
- })
- }
-
- isEnd := len(prs) < perPage
- if !g.pagination {
- isEnd = len(prs) == 0
- }
- return allPRs, isEnd, nil
-}
-
-// GetReviews returns pull requests review
-func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
- if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil {
- log.Info("GiteaDownloader: instance to old, skip GetReviews")
- return nil, nil
- }
-
- var allReviews = make([]*base.Review, 0, g.maxPerPage)
-
- for i := 1; ; i++ {
- // make sure gitea can shutdown gracefully
- select {
- case <-g.ctx.Done():
- return nil, nil
- default:
- }
-
- prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, context.ForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{
- Page: i,
- PageSize: g.maxPerPage,
- }})
- if err != nil {
- return nil, err
- }
-
- for _, pr := range prl {
-
- rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, context.ForeignID(), pr.ID)
- if err != nil {
- return nil, err
- }
- var reviewComments []*base.ReviewComment
- for i := range rcl {
- line := int(rcl[i].LineNum)
- if rcl[i].OldLineNum > 0 {
- line = int(rcl[i].OldLineNum) * -1
- }
-
- reviewComments = append(reviewComments, &base.ReviewComment{
- ID: rcl[i].ID,
- Content: rcl[i].Body,
- TreePath: rcl[i].Path,
- DiffHunk: rcl[i].DiffHunk,
- Line: line,
- CommitID: rcl[i].CommitID,
- PosterID: rcl[i].Reviewer.ID,
- CreatedAt: rcl[i].Created,
- UpdatedAt: rcl[i].Updated,
- })
- }
-
- allReviews = append(allReviews, &base.Review{
- ID: pr.ID,
- IssueIndex: context.LocalID(),
- ReviewerID: pr.Reviewer.ID,
- ReviewerName: pr.Reviewer.UserName,
- Official: pr.Official,
- CommitID: pr.CommitID,
- Content: pr.Body,
- CreatedAt: pr.Submitted,
- State: string(pr.State),
- Comments: reviewComments,
- })
- }
-
- if len(prl) < g.maxPerPage {
- break
- }
- }
- return allReviews, nil
-}
+++ /dev/null
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "net/http"
- "os"
- "sort"
- "testing"
- "time"
-
- "code.gitea.io/gitea/modules/migrations/base"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestGiteaDownloadRepo(t *testing.T) {
- // Skip tests if Gitea token is not found
- giteaToken := os.Getenv("GITEA_TOKEN")
- if giteaToken == "" {
- t.Skip("skipped test because GITEA_TOKEN was not in the environment")
- }
-
- resp, err := http.Get("https://gitea.com/gitea")
- if err != nil || resp.StatusCode != 200 {
- t.Skipf("Can't reach https://gitea.com, skipping %s", t.Name())
- }
-
- downloader, err := NewGiteaDownloader(context.Background(), "https://gitea.com", "gitea/test_repo", "", "", giteaToken)
- if downloader == nil {
- t.Fatal("NewGitlabDownloader is nil")
- }
- if !assert.NoError(t, err) {
- t.Fatal("NewGitlabDownloader error occur")
- }
-
- repo, err := downloader.GetRepoInfo()
- assert.NoError(t, err)
- assertRepositoryEqual(t, &base.Repository{
- Name: "test_repo",
- Owner: "gitea",
- IsPrivate: false,
- Description: "Test repository for testing migration from gitea to gitea",
- CloneURL: "https://gitea.com/gitea/test_repo.git",
- OriginalURL: "https://gitea.com/gitea/test_repo",
- DefaultBranch: "master",
- }, repo)
-
- topics, err := downloader.GetTopics()
- assert.NoError(t, err)
- sort.Strings(topics)
- assert.EqualValues(t, []string{"ci", "gitea", "migration", "test"}, topics)
-
- labels, err := downloader.GetLabels()
- assert.NoError(t, err)
- assertLabelsEqual(t, []*base.Label{
- {
- Name: "Bug",
- Color: "e11d21",
- },
- {
- Name: "Enhancement",
- Color: "207de5",
- },
- {
- Name: "Feature",
- Color: "0052cc",
- Description: "a feature request",
- },
- {
- Name: "Invalid",
- Color: "d4c5f9",
- },
- {
- Name: "Question",
- Color: "fbca04",
- },
- {
- Name: "Valid",
- Color: "53e917",
- },
- }, labels)
-
- milestones, err := downloader.GetMilestones()
- assert.NoError(t, err)
- assertMilestonesEqual(t, []*base.Milestone{
- {
- Title: "V2 Finalize",
- Created: time.Unix(0, 0),
- Deadline: timePtr(time.Unix(1599263999, 0)),
- Updated: timePtr(time.Unix(0, 0)),
- State: "open",
- },
- {
- Title: "V1",
- Description: "Generate Content",
- Created: time.Unix(0, 0),
- Updated: timePtr(time.Unix(0, 0)),
- Closed: timePtr(time.Unix(1598985406, 0)),
- State: "closed",
- },
- }, milestones)
-
- releases, err := downloader.GetReleases()
- assert.NoError(t, err)
- assertReleasesEqual(t, []*base.Release{
- {
- Name: "Second Release",
- TagName: "v2-rc1",
- TargetCommitish: "master",
- Body: "this repo has:\r\n* reactions\r\n* wiki\r\n* issues (open/closed)\r\n* pulls (open/closed/merged) (external/internal)\r\n* pull reviews\r\n* projects\r\n* milestones\r\n* labels\r\n* releases\r\n\r\nto test migration against",
- Draft: false,
- Prerelease: true,
- Created: time.Date(2020, 9, 1, 18, 2, 43, 0, time.UTC),
- Published: time.Date(2020, 9, 1, 18, 2, 43, 0, time.UTC),
- PublisherID: 689,
- PublisherName: "6543",
- PublisherEmail: "6543@obermui.de",
- },
- {
- Name: "First Release",
- TagName: "V1",
- TargetCommitish: "master",
- Body: "as title",
- Draft: false,
- Prerelease: false,
- Created: time.Date(2020, 9, 1, 17, 30, 32, 0, time.UTC),
- Published: time.Date(2020, 9, 1, 17, 30, 32, 0, time.UTC),
- PublisherID: 689,
- PublisherName: "6543",
- PublisherEmail: "6543@obermui.de",
- },
- }, releases)
-
- issues, isEnd, err := downloader.GetIssues(1, 50)
- assert.NoError(t, err)
- assert.True(t, isEnd)
- assert.Len(t, issues, 7)
- assert.EqualValues(t, "open", issues[0].State)
-
- issues, isEnd, err = downloader.GetIssues(3, 2)
- assert.NoError(t, err)
- assert.False(t, isEnd)
-
- assertIssuesEqual(t, []*base.Issue{
- {
- Number: 4,
- Title: "what is this repo about?",
- Content: "",
- Milestone: "V1",
- PosterID: -1,
- PosterName: "Ghost",
- PosterEmail: "",
- State: "closed",
- IsLocked: true,
- Created: time.Unix(1598975321, 0),
- Updated: time.Unix(1598975400, 0),
- Labels: []*base.Label{{
- Name: "Question",
- Color: "fbca04",
- Description: "",
- }},
- Reactions: []*base.Reaction{
- {
- UserID: 689,
- UserName: "6543",
- Content: "gitea",
- },
- {
- UserID: 689,
- UserName: "6543",
- Content: "laugh",
- },
- },
- Closed: timePtr(time.Date(2020, 9, 1, 15, 49, 34, 0, time.UTC)),
- },
- {
- Number: 2,
- Title: "Spam",
- Content: ":(",
- Milestone: "",
- PosterID: 689,
- PosterName: "6543",
- PosterEmail: "6543@obermui.de",
- State: "closed",
- IsLocked: false,
- Created: time.Unix(1598919780, 0),
- Updated: time.Unix(1598969497, 0),
- Labels: []*base.Label{{
- Name: "Invalid",
- Color: "d4c5f9",
- Description: "",
- }},
- Closed: timePtr(time.Unix(1598969497, 0)),
- },
- }, issues)
-
- comments, _, err := downloader.GetComments(base.GetCommentOptions{
- Context: base.BasicIssueContext(4),
- })
- assert.NoError(t, err)
- assertCommentsEqual(t, []*base.Comment{
- {
- IssueIndex: 4,
- PosterID: 689,
- PosterName: "6543",
- PosterEmail: "6543@obermui.de",
- Created: time.Unix(1598975370, 0),
- Updated: time.Unix(1599070865, 0),
- Content: "a really good question!\n\nIt is the used as TESTSET for gitea2gitea repo migration function",
- },
- {
- IssueIndex: 4,
- PosterID: -1,
- PosterName: "Ghost",
- PosterEmail: "",
- Created: time.Unix(1598975393, 0),
- Updated: time.Unix(1598975393, 0),
- Content: "Oh!",
- },
- }, comments)
-
- prs, isEnd, err := downloader.GetPullRequests(1, 50)
- assert.NoError(t, err)
- assert.True(t, isEnd)
- assert.Len(t, prs, 6)
- prs, isEnd, err = downloader.GetPullRequests(1, 3)
- assert.NoError(t, err)
- assert.False(t, isEnd)
- assert.Len(t, prs, 3)
- assertPullRequestEqual(t, &base.PullRequest{
- Number: 12,
- PosterID: 689,
- PosterName: "6543",
- PosterEmail: "6543@obermui.de",
- Title: "Dont Touch",
- Content: "\r\nadd dont touch note",
- Milestone: "V2 Finalize",
- State: "closed",
- IsLocked: false,
- Created: time.Unix(1598982759, 0),
- Updated: time.Unix(1599023425, 0),
- Closed: timePtr(time.Unix(1598982934, 0)),
- Assignees: []string{"techknowlogick"},
- Base: base.PullRequestBranch{
- CloneURL: "",
- Ref: "master",
- SHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
- RepoName: "test_repo",
- OwnerName: "gitea",
- },
- Head: base.PullRequestBranch{
- CloneURL: "https://gitea.com/6543-forks/test_repo.git",
- Ref: "refs/pull/12/head",
- SHA: "b6ab5d9ae000b579a5fff03f92c486da4ddf48b6",
- RepoName: "test_repo",
- OwnerName: "6543-forks",
- },
- Merged: true,
- MergedTime: timePtr(time.Unix(1598982934, 0)),
- MergeCommitSHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
- PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch",
- }, prs[1])
-
- reviews, err := downloader.GetReviews(base.BasicIssueContext(7))
- assert.NoError(t, err)
- assertReviewsEqual(t, []*base.Review{
- {
- ID: 1770,
- IssueIndex: 7,
- ReviewerID: 689,
- ReviewerName: "6543",
- CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
- CreatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
- State: "COMMENT", // TODO
- Comments: []*base.ReviewComment{
- {
- ID: 116561,
- InReplyTo: 0,
- Content: "is one `\\newline` to less?",
- TreePath: "README.md",
- DiffHunk: "@@ -2,3 +2,3 @@\n \n-Test repository for testing migration from gitea 2 gitea\n\\ No newline at end of file\n+Test repository for testing migration from gitea 2 gitea",
- Position: 0,
- Line: 4,
- CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
- PosterID: 689,
- Reactions: nil,
- CreatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
- UpdatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
- },
- },
- },
- {
- ID: 1771,
- IssueIndex: 7,
- ReviewerID: 9,
- ReviewerName: "techknowlogick",
- CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
- CreatedAt: time.Date(2020, 9, 1, 17, 6, 47, 0, time.UTC),
- State: "REQUEST_CHANGES", // TODO
- Content: "I think this needs some changes",
- },
- {
- ID: 1772,
- IssueIndex: 7,
- ReviewerID: 9,
- ReviewerName: "techknowlogick",
- CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
- CreatedAt: time.Date(2020, 9, 1, 17, 19, 51, 0, time.UTC),
- State: base.ReviewStateApproved,
- Official: true,
- Content: "looks good",
- },
- }, reviews)
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "fmt"
- "io"
- "os"
- "path/filepath"
- "strings"
- "sync"
- "time"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/models/db"
- "code.gitea.io/gitea/modules/git"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations/base"
- repo_module "code.gitea.io/gitea/modules/repository"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/storage"
- "code.gitea.io/gitea/modules/structs"
- "code.gitea.io/gitea/modules/timeutil"
- "code.gitea.io/gitea/modules/uri"
- "code.gitea.io/gitea/services/pull"
-
- gouuid "github.com/google/uuid"
-)
-
-var (
- _ base.Uploader = &GiteaLocalUploader{}
-)
-
-// GiteaLocalUploader implements an Uploader to gitea sites
-type GiteaLocalUploader struct {
- ctx context.Context
- doer *models.User
- repoOwner string
- repoName string
- repo *models.Repository
- labels sync.Map
- milestones sync.Map
- issues sync.Map
- gitRepo *git.Repository
- prHeadCache map[string]struct{}
- userMap map[int64]int64 // external user id mapping to user id
- prCache map[int64]*models.PullRequest
- gitServiceType structs.GitServiceType
-}
-
-// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
-func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
- return &GiteaLocalUploader{
- ctx: ctx,
- doer: doer,
- repoOwner: repoOwner,
- repoName: repoName,
- prHeadCache: make(map[string]struct{}),
- userMap: make(map[int64]int64),
- prCache: make(map[int64]*models.PullRequest),
- }
-}
-
-// MaxBatchInsertSize returns the table's max batch insert size
-func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
- switch tp {
- case "issue":
- return db.MaxBatchInsertSize(new(models.Issue))
- case "comment":
- return db.MaxBatchInsertSize(new(models.Comment))
- case "milestone":
- return db.MaxBatchInsertSize(new(models.Milestone))
- case "label":
- return db.MaxBatchInsertSize(new(models.Label))
- case "release":
- return db.MaxBatchInsertSize(new(models.Release))
- case "pullrequest":
- return db.MaxBatchInsertSize(new(models.PullRequest))
- }
- return 10
-}
-
-// CreateRepo creates a repository
-func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
- owner, err := models.GetUserByName(g.repoOwner)
- if err != nil {
- return err
- }
-
- var r *models.Repository
- if opts.MigrateToRepoID <= 0 {
- r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{
- Name: g.repoName,
- Description: repo.Description,
- OriginalURL: repo.OriginalURL,
- GitServiceType: opts.GitServiceType,
- IsPrivate: opts.Private,
- IsMirror: opts.Mirror,
- Status: models.RepositoryBeingMigrated,
- })
- } else {
- r, err = models.GetRepositoryByID(opts.MigrateToRepoID)
- }
- if err != nil {
- return err
- }
- r.DefaultBranch = repo.DefaultBranch
-
- r, err = repo_module.MigrateRepositoryGitData(g.ctx, owner, r, base.MigrateOptions{
- RepoName: g.repoName,
- Description: repo.Description,
- OriginalURL: repo.OriginalURL,
- GitServiceType: opts.GitServiceType,
- Mirror: repo.IsMirror,
- LFS: opts.LFS,
- LFSEndpoint: opts.LFSEndpoint,
- CloneAddr: repo.CloneURL,
- Private: repo.IsPrivate,
- Wiki: opts.Wiki,
- Releases: opts.Releases, // if didn't get releases, then sync them from tags
- MirrorInterval: opts.MirrorInterval,
- })
-
- g.repo = r
- if err != nil {
- return err
- }
- g.gitRepo, err = git.OpenRepository(r.RepoPath())
- return err
-}
-
-// Close closes this uploader
-func (g *GiteaLocalUploader) Close() {
- if g.gitRepo != nil {
- g.gitRepo.Close()
- }
-}
-
-// CreateTopics creates topics
-func (g *GiteaLocalUploader) CreateTopics(topics ...string) error {
- // ignore topics to long for the db
- c := 0
- for i := range topics {
- if len(topics[i]) <= 50 {
- topics[c] = topics[i]
- c++
- }
- }
- topics = topics[:c]
- return models.SaveTopics(g.repo.ID, topics...)
-}
-
-// CreateMilestones creates milestones
-func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error {
- var mss = make([]*models.Milestone, 0, len(milestones))
- for _, milestone := range milestones {
- var deadline timeutil.TimeStamp
- if milestone.Deadline != nil {
- deadline = timeutil.TimeStamp(milestone.Deadline.Unix())
- }
- if deadline == 0 {
- deadline = timeutil.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.DefaultUILocation).Unix())
- }
-
- if milestone.Created.IsZero() {
- if milestone.Updated != nil {
- milestone.Created = *milestone.Updated
- } else if milestone.Deadline != nil {
- milestone.Created = *milestone.Deadline
- } else {
- milestone.Created = time.Now()
- }
- }
- if milestone.Updated == nil || milestone.Updated.IsZero() {
- milestone.Updated = &milestone.Created
- }
-
- var ms = models.Milestone{
- RepoID: g.repo.ID,
- Name: milestone.Title,
- Content: milestone.Description,
- IsClosed: milestone.State == "closed",
- CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()),
- UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()),
- DeadlineUnix: deadline,
- }
- if ms.IsClosed && milestone.Closed != nil {
- ms.ClosedDateUnix = timeutil.TimeStamp(milestone.Closed.Unix())
- }
- mss = append(mss, &ms)
- }
-
- err := models.InsertMilestones(mss...)
- if err != nil {
- return err
- }
-
- for _, ms := range mss {
- g.milestones.Store(ms.Name, ms.ID)
- }
- return nil
-}
-
-// CreateLabels creates labels
-func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
- var lbs = make([]*models.Label, 0, len(labels))
- for _, label := range labels {
- lbs = append(lbs, &models.Label{
- RepoID: g.repo.ID,
- Name: label.Name,
- Description: label.Description,
- Color: fmt.Sprintf("#%s", label.Color),
- })
- }
-
- err := models.NewLabels(lbs...)
- if err != nil {
- return err
- }
- for _, lb := range lbs {
- g.labels.Store(lb.Name, lb)
- }
- return nil
-}
-
-// CreateReleases creates releases
-func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
- var rels = make([]*models.Release, 0, len(releases))
- for _, release := range releases {
- if release.Created.IsZero() {
- if !release.Published.IsZero() {
- release.Created = release.Published
- } else {
- release.Created = time.Now()
- }
- }
-
- var rel = models.Release{
- RepoID: g.repo.ID,
- TagName: release.TagName,
- LowerTagName: strings.ToLower(release.TagName),
- Target: release.TargetCommitish,
- Title: release.Name,
- Sha1: release.TargetCommitish,
- Note: release.Body,
- IsDraft: release.Draft,
- IsPrerelease: release.Prerelease,
- IsTag: false,
- CreatedUnix: timeutil.TimeStamp(release.Created.Unix()),
- }
-
- userid, ok := g.userMap[release.PublisherID]
- tp := g.gitServiceType.Name()
- if !ok && tp != "" {
- var err error
- userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID))
- if err != nil {
- log.Error("GetUserIDByExternalUserID: %v", err)
- }
- if userid > 0 {
- g.userMap[release.PublisherID] = userid
- }
- }
-
- if userid > 0 {
- rel.PublisherID = userid
- } else {
- rel.PublisherID = g.doer.ID
- rel.OriginalAuthor = release.PublisherName
- rel.OriginalAuthorID = release.PublisherID
- }
-
- // calc NumCommits if no draft
- if !release.Draft {
- commit, err := g.gitRepo.GetTagCommit(rel.TagName)
- if err != nil {
- return fmt.Errorf("GetTagCommit[%v]: %v", rel.TagName, err)
- }
- rel.NumCommits, err = commit.CommitsCount()
- if err != nil {
- return fmt.Errorf("CommitsCount: %v", err)
- }
- }
-
- for _, asset := range release.Assets {
- if asset.Created.IsZero() {
- if !asset.Updated.IsZero() {
- asset.Created = asset.Updated
- } else {
- asset.Created = release.Created
- }
- }
- var attach = models.Attachment{
- UUID: gouuid.New().String(),
- Name: asset.Name,
- DownloadCount: int64(*asset.DownloadCount),
- Size: int64(*asset.Size),
- CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()),
- }
-
- // download attachment
- err := func() error {
- // asset.DownloadURL maybe a local file
- var rc io.ReadCloser
- var err error
- if asset.DownloadFunc != nil {
- rc, err = asset.DownloadFunc()
- if err != nil {
- return err
- }
- } else if asset.DownloadURL != nil {
- rc, err = uri.Open(*asset.DownloadURL)
- if err != nil {
- return err
- }
- }
- if rc == nil {
- return nil
- }
- _, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size))
- rc.Close()
- return err
- }()
- if err != nil {
- return err
- }
-
- rel.Attachments = append(rel.Attachments, &attach)
- }
-
- rels = append(rels, &rel)
- }
-
- return models.InsertReleases(rels...)
-}
-
-// SyncTags syncs releases with tags in the database
-func (g *GiteaLocalUploader) SyncTags() error {
- return repo_module.SyncReleasesWithTags(g.repo, g.gitRepo)
-}
-
-// CreateIssues creates issues
-func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
- var iss = make([]*models.Issue, 0, len(issues))
- for _, issue := range issues {
- var labels []*models.Label
- for _, label := range issue.Labels {
- lb, ok := g.labels.Load(label.Name)
- if ok {
- labels = append(labels, lb.(*models.Label))
- }
- }
-
- var milestoneID int64
- if issue.Milestone != "" {
- milestone, ok := g.milestones.Load(issue.Milestone)
- if ok {
- milestoneID = milestone.(int64)
- }
- }
-
- if issue.Created.IsZero() {
- if issue.Closed != nil {
- issue.Created = *issue.Closed
- } else {
- issue.Created = time.Now()
- }
- }
- if issue.Updated.IsZero() {
- if issue.Closed != nil {
- issue.Updated = *issue.Closed
- } else {
- issue.Updated = time.Now()
- }
- }
-
- var is = models.Issue{
- RepoID: g.repo.ID,
- Repo: g.repo,
- Index: issue.Number,
- Title: issue.Title,
- Content: issue.Content,
- Ref: issue.Ref,
- IsClosed: issue.State == "closed",
- IsLocked: issue.IsLocked,
- MilestoneID: milestoneID,
- Labels: labels,
- CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()),
- UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()),
- }
-
- userid, ok := g.userMap[issue.PosterID]
- tp := g.gitServiceType.Name()
- if !ok && tp != "" {
- var err error
- userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID))
- if err != nil {
- log.Error("GetUserIDByExternalUserID: %v", err)
- }
- if userid > 0 {
- g.userMap[issue.PosterID] = userid
- }
- }
-
- if userid > 0 {
- is.PosterID = userid
- } else {
- is.PosterID = g.doer.ID
- is.OriginalAuthor = issue.PosterName
- is.OriginalAuthorID = issue.PosterID
- }
-
- if issue.Closed != nil {
- is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix())
- }
- // add reactions
- for _, reaction := range issue.Reactions {
- userid, ok := g.userMap[reaction.UserID]
- if !ok && tp != "" {
- var err error
- userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
- if err != nil {
- log.Error("GetUserIDByExternalUserID: %v", err)
- }
- if userid > 0 {
- g.userMap[reaction.UserID] = userid
- }
- }
- var res = models.Reaction{
- Type: reaction.Content,
- CreatedUnix: timeutil.TimeStampNow(),
- }
- if userid > 0 {
- res.UserID = userid
- } else {
- res.UserID = g.doer.ID
- res.OriginalAuthorID = reaction.UserID
- res.OriginalAuthor = reaction.UserName
- }
- is.Reactions = append(is.Reactions, &res)
- }
- iss = append(iss, &is)
- }
-
- if len(iss) > 0 {
- if err := models.InsertIssues(iss...); err != nil {
- return err
- }
-
- for _, is := range iss {
- g.issues.Store(is.Index, is)
- }
- }
-
- return nil
-}
-
-// CreateComments creates comments of issues
-func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
- var cms = make([]*models.Comment, 0, len(comments))
- for _, comment := range comments {
- var issue *models.Issue
- issueInter, ok := g.issues.Load(comment.IssueIndex)
- if !ok {
- var err error
- issue, err = models.GetIssueByIndex(g.repo.ID, comment.IssueIndex)
- if err != nil {
- return err
- }
- g.issues.Store(comment.IssueIndex, issue)
- } else {
- issue = issueInter.(*models.Issue)
- }
-
- userid, ok := g.userMap[comment.PosterID]
- tp := g.gitServiceType.Name()
- if !ok && tp != "" {
- var err error
- userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID))
- if err != nil {
- log.Error("GetUserIDByExternalUserID: %v", err)
- }
- if userid > 0 {
- g.userMap[comment.PosterID] = userid
- }
- }
-
- if comment.Created.IsZero() {
- comment.Created = time.Unix(int64(issue.CreatedUnix), 0)
- }
- if comment.Updated.IsZero() {
- comment.Updated = comment.Created
- }
-
- cm := models.Comment{
- IssueID: issue.ID,
- Type: models.CommentTypeComment,
- Content: comment.Content,
- CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()),
- UpdatedUnix: timeutil.TimeStamp(comment.Updated.Unix()),
- }
-
- if userid > 0 {
- cm.PosterID = userid
- } else {
- cm.PosterID = g.doer.ID
- cm.OriginalAuthor = comment.PosterName
- cm.OriginalAuthorID = comment.PosterID
- }
-
- // add reactions
- for _, reaction := range comment.Reactions {
- userid, ok := g.userMap[reaction.UserID]
- if !ok && tp != "" {
- var err error
- userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
- if err != nil {
- log.Error("GetUserIDByExternalUserID: %v", err)
- }
- if userid > 0 {
- g.userMap[reaction.UserID] = userid
- }
- }
- var res = models.Reaction{
- Type: reaction.Content,
- CreatedUnix: timeutil.TimeStampNow(),
- }
- if userid > 0 {
- res.UserID = userid
- } else {
- res.UserID = g.doer.ID
- res.OriginalAuthorID = reaction.UserID
- res.OriginalAuthor = reaction.UserName
- }
- cm.Reactions = append(cm.Reactions, &res)
- }
-
- cms = append(cms, &cm)
- }
-
- if len(cms) == 0 {
- return nil
- }
- return models.InsertIssueComments(cms)
-}
-
-// CreatePullRequests creates pull requests
-func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error {
- var gprs = make([]*models.PullRequest, 0, len(prs))
- for _, pr := range prs {
- gpr, err := g.newPullRequest(pr)
- if err != nil {
- return err
- }
-
- userid, ok := g.userMap[pr.PosterID]
- tp := g.gitServiceType.Name()
- if !ok && tp != "" {
- var err error
- userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
- if err != nil {
- log.Error("GetUserIDByExternalUserID: %v", err)
- }
- if userid > 0 {
- g.userMap[pr.PosterID] = userid
- }
- }
-
- if userid > 0 {
- gpr.Issue.PosterID = userid
- } else {
- gpr.Issue.PosterID = g.doer.ID
- gpr.Issue.OriginalAuthor = pr.PosterName
- gpr.Issue.OriginalAuthorID = pr.PosterID
- }
-
- gprs = append(gprs, gpr)
- }
- if err := models.InsertPullRequests(gprs...); err != nil {
- return err
- }
- for _, pr := range gprs {
- g.issues.Store(pr.Issue.Index, pr.Issue)
- pull.AddToTaskQueue(pr)
- }
- return nil
-}
-
-func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) {
- var labels []*models.Label
- for _, label := range pr.Labels {
- lb, ok := g.labels.Load(label.Name)
- if ok {
- labels = append(labels, lb.(*models.Label))
- }
- }
-
- var milestoneID int64
- if pr.Milestone != "" {
- milestone, ok := g.milestones.Load(pr.Milestone)
- if ok {
- milestoneID = milestone.(int64)
- }
- }
-
- // download patch file
- err := func() error {
- if pr.PatchURL == "" {
- return nil
- }
- // pr.PatchURL maybe a local file
- ret, err := uri.Open(pr.PatchURL)
- if err != nil {
- return err
- }
- defer ret.Close()
- pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
- if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
- return err
- }
- f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)))
- if err != nil {
- return err
- }
- defer f.Close()
- _, err = io.Copy(f, ret)
- return err
- }()
- if err != nil {
- return nil, err
- }
-
- // set head information
- pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
- if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
- return nil, err
- }
- p, err := os.Create(filepath.Join(pullHead, "head"))
- if err != nil {
- return nil, err
- }
- _, err = p.WriteString(pr.Head.SHA)
- p.Close()
- if err != nil {
- return nil, err
- }
-
- var head = "unknown repository"
- if pr.IsForkPullRequest() && pr.State != "closed" {
- if pr.Head.OwnerName != "" {
- remote := pr.Head.OwnerName
- _, ok := g.prHeadCache[remote]
- if !ok {
- // git remote add
- err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
- if err != nil {
- log.Error("AddRemote failed: %s", err)
- } else {
- g.prHeadCache[remote] = struct{}{}
- ok = true
- }
- }
-
- if ok {
- _, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.repo.RepoPath())
- if err != nil {
- log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
- } else {
- headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
- if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
- return nil, err
- }
- b, err := os.Create(headBranch)
- if err != nil {
- return nil, err
- }
- _, err = b.WriteString(pr.Head.SHA)
- b.Close()
- if err != nil {
- return nil, err
- }
- head = pr.Head.OwnerName + "/" + pr.Head.Ref
- }
- }
- }
- } else {
- head = pr.Head.Ref
- // Ensure the closed PR SHA still points to an existing ref
- _, err = git.NewCommand("rev-list", "--quiet", "-1", pr.Head.SHA).RunInDir(g.repo.RepoPath())
- if err != nil {
- if pr.Head.SHA != "" {
- // Git update-ref remove bad references with a relative path
- log.Warn("Deprecated local head, removing : %v", pr.Head.SHA)
- relPath := pr.GetGitRefName()
- _, err = git.NewCommand("update-ref", "--no-deref", "-d", relPath).RunInDir(g.repo.RepoPath())
- } else {
- // The SHA is empty, remove the head file
- log.Warn("Empty reference, removing : %v", pullHead)
- err = os.Remove(filepath.Join(pullHead, "head"))
- }
- if err != nil {
- log.Error("Cannot remove local head ref, %v", err)
- }
- }
- }
-
- if pr.Created.IsZero() {
- if pr.Closed != nil {
- pr.Created = *pr.Closed
- } else if pr.MergedTime != nil {
- pr.Created = *pr.MergedTime
- } else {
- pr.Created = time.Now()
- }
- }
- if pr.Updated.IsZero() {
- pr.Updated = pr.Created
- }
-
- var issue = models.Issue{
- RepoID: g.repo.ID,
- Repo: g.repo,
- Title: pr.Title,
- Index: pr.Number,
- Content: pr.Content,
- MilestoneID: milestoneID,
- IsPull: true,
- IsClosed: pr.State == "closed",
- IsLocked: pr.IsLocked,
- Labels: labels,
- CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()),
- UpdatedUnix: timeutil.TimeStamp(pr.Updated.Unix()),
- }
-
- tp := g.gitServiceType.Name()
-
- userid, ok := g.userMap[pr.PosterID]
- if !ok && tp != "" {
- var err error
- userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
- if err != nil {
- log.Error("GetUserIDByExternalUserID: %v", err)
- }
- if userid > 0 {
- g.userMap[pr.PosterID] = userid
- }
- }
-
- if userid > 0 {
- issue.PosterID = userid
- } else {
- issue.PosterID = g.doer.ID
- issue.OriginalAuthor = pr.PosterName
- issue.OriginalAuthorID = pr.PosterID
- }
-
- // add reactions
- for _, reaction := range pr.Reactions {
- userid, ok := g.userMap[reaction.UserID]
- if !ok && tp != "" {
- var err error
- userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
- if err != nil {
- log.Error("GetUserIDByExternalUserID: %v", err)
- }
- if userid > 0 {
- g.userMap[reaction.UserID] = userid
- }
- }
- var res = models.Reaction{
- Type: reaction.Content,
- CreatedUnix: timeutil.TimeStampNow(),
- }
- if userid > 0 {
- res.UserID = userid
- } else {
- res.UserID = g.doer.ID
- res.OriginalAuthorID = reaction.UserID
- res.OriginalAuthor = reaction.UserName
- }
- issue.Reactions = append(issue.Reactions, &res)
- }
-
- var pullRequest = models.PullRequest{
- HeadRepoID: g.repo.ID,
- HeadBranch: head,
- BaseRepoID: g.repo.ID,
- BaseBranch: pr.Base.Ref,
- MergeBase: pr.Base.SHA,
- Index: pr.Number,
- HasMerged: pr.Merged,
-
- Issue: &issue,
- }
-
- if pullRequest.Issue.IsClosed && pr.Closed != nil {
- pullRequest.Issue.ClosedUnix = timeutil.TimeStamp(pr.Closed.Unix())
- }
- if pullRequest.HasMerged && pr.MergedTime != nil {
- pullRequest.MergedUnix = timeutil.TimeStamp(pr.MergedTime.Unix())
- pullRequest.MergedCommitID = pr.MergeCommitSHA
- pullRequest.MergerID = g.doer.ID
- }
-
- // TODO: assignees
-
- return &pullRequest, nil
-}
-
-func convertReviewState(state string) models.ReviewType {
- switch state {
- case base.ReviewStatePending:
- return models.ReviewTypePending
- case base.ReviewStateApproved:
- return models.ReviewTypeApprove
- case base.ReviewStateChangesRequested:
- return models.ReviewTypeReject
- case base.ReviewStateCommented:
- return models.ReviewTypeComment
- default:
- return models.ReviewTypePending
- }
-}
-
-// CreateReviews create pull request reviews
-func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
- var cms = make([]*models.Review, 0, len(reviews))
- for _, review := range reviews {
- var issue *models.Issue
- issueInter, ok := g.issues.Load(review.IssueIndex)
- if !ok {
- var err error
- issue, err = models.GetIssueByIndex(g.repo.ID, review.IssueIndex)
- if err != nil {
- return err
- }
- g.issues.Store(review.IssueIndex, issue)
- } else {
- issue = issueInter.(*models.Issue)
- }
-
- userid, ok := g.userMap[review.ReviewerID]
- tp := g.gitServiceType.Name()
- if !ok && tp != "" {
- var err error
- userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", review.ReviewerID))
- if err != nil {
- log.Error("GetUserIDByExternalUserID: %v", err)
- }
- if userid > 0 {
- g.userMap[review.ReviewerID] = userid
- }
- }
-
- if review.CreatedAt.IsZero() {
- review.CreatedAt = time.Unix(int64(issue.CreatedUnix), 0)
- }
-
- var cm = models.Review{
- Type: convertReviewState(review.State),
- IssueID: issue.ID,
- Content: review.Content,
- Official: review.Official,
- CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
- UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
- }
-
- if userid > 0 {
- cm.ReviewerID = userid
- } else {
- cm.ReviewerID = g.doer.ID
- cm.OriginalAuthor = review.ReviewerName
- cm.OriginalAuthorID = review.ReviewerID
- }
-
- // get pr
- pr, ok := g.prCache[issue.ID]
- if !ok {
- var err error
- pr, err = models.GetPullRequestByIssueIDWithNoAttributes(issue.ID)
- if err != nil {
- return err
- }
- g.prCache[issue.ID] = pr
- }
-
- for _, comment := range review.Comments {
- line := comment.Line
- if line != 0 {
- comment.Position = 1
- } else {
- _, _, line, _ = git.ParseDiffHunkString(comment.DiffHunk)
- }
- headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName())
- if err != nil {
- log.Warn("GetRefCommitID[%s]: %v, the review comment will be ignored", pr.GetGitRefName(), err)
- continue
- }
-
- var patch string
- reader, writer := io.Pipe()
- defer func() {
- _ = reader.Close()
- _ = writer.Close()
- }()
- go func() {
- if err := git.GetRepoRawDiffForFile(g.gitRepo, pr.MergeBase, headCommitID, git.RawDiffNormal, comment.TreePath, writer); err != nil {
- // We should ignore the error since the commit maybe removed when force push to the pull request
- log.Warn("GetRepoRawDiffForFile failed when migrating [%s, %s, %s, %s]: %v", g.gitRepo.Path, pr.MergeBase, headCommitID, comment.TreePath, err)
- }
- _ = writer.Close()
- }()
-
- patch, _ = git.CutDiffAroundLine(reader, int64((&models.Comment{Line: int64(line + comment.Position - 1)}).UnsignedLine()), line < 0, setting.UI.CodeCommentLines)
-
- if comment.CreatedAt.IsZero() {
- comment.CreatedAt = review.CreatedAt
- }
- if comment.UpdatedAt.IsZero() {
- comment.UpdatedAt = comment.CreatedAt
- }
-
- var c = models.Comment{
- Type: models.CommentTypeCode,
- PosterID: comment.PosterID,
- IssueID: issue.ID,
- Content: comment.Content,
- Line: int64(line + comment.Position - 1),
- TreePath: comment.TreePath,
- CommitSHA: comment.CommitID,
- Patch: patch,
- CreatedUnix: timeutil.TimeStamp(comment.CreatedAt.Unix()),
- UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()),
- }
-
- if userid > 0 {
- c.PosterID = userid
- } else {
- c.PosterID = g.doer.ID
- c.OriginalAuthor = review.ReviewerName
- c.OriginalAuthorID = review.ReviewerID
- }
-
- cm.Comments = append(cm.Comments, &c)
- }
-
- cms = append(cms, &cm)
- }
-
- return models.InsertReviews(cms)
-}
-
-// Rollback when migrating failed, this will rollback all the changes.
-func (g *GiteaLocalUploader) Rollback() error {
- if g.repo != nil && g.repo.ID > 0 {
- g.gitRepo.Close()
- if err := models.DeleteRepository(g.doer, g.repo.OwnerID, g.repo.ID); err != nil {
- return err
- }
- }
- return nil
-}
-
-// Finish when migrating success, this will do some status update things.
-func (g *GiteaLocalUploader) Finish() error {
- if g.repo == nil || g.repo.ID <= 0 {
- return ErrRepoNotCreated
- }
-
- // update issue_index
- if err := models.RecalculateIssueIndexForRepo(g.repo.ID); err != nil {
- return err
- }
-
- g.repo.Status = models.RepositoryReady
- return models.UpdateRepositoryCols(g.repo, "status")
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "testing"
- "time"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/models/db"
- "code.gitea.io/gitea/models/unittest"
- "code.gitea.io/gitea/modules/graceful"
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/structs"
- "code.gitea.io/gitea/modules/util"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestGiteaUploadRepo(t *testing.T) {
- // FIXME: Since no accesskey or user/password will trigger rate limit of github, just skip
- t.Skip()
-
- unittest.PrepareTestEnv(t)
-
- user := unittest.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
-
- var (
- downloader = NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", "", "go-xorm", "builder")
- repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05")
- uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
- )
-
- err := migrateRepository(downloader, uploader, base.MigrateOptions{
- CloneAddr: "https://github.com/go-xorm/builder",
- RepoName: repoName,
- AuthUsername: "",
-
- Wiki: true,
- Issues: true,
- Milestones: true,
- Labels: true,
- Releases: true,
- Comments: true,
- PullRequests: true,
- Private: true,
- Mirror: false,
- }, nil)
- assert.NoError(t, err)
-
- repo := unittest.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)
- assert.True(t, repo.HasWiki())
- assert.EqualValues(t, models.RepositoryReady, repo.Status)
-
- milestones, _, err := models.GetMilestones(models.GetMilestonesOption{
- RepoID: repo.ID,
- State: structs.StateOpen,
- })
- assert.NoError(t, err)
- assert.Len(t, milestones, 1)
-
- milestones, _, err = models.GetMilestones(models.GetMilestonesOption{
- RepoID: repo.ID,
- State: structs.StateClosed,
- })
- assert.NoError(t, err)
- assert.Empty(t, milestones)
-
- labels, err := models.GetLabelsByRepoID(repo.ID, "", db.ListOptions{})
- assert.NoError(t, err)
- assert.Len(t, labels, 12)
-
- releases, err := models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
- ListOptions: db.ListOptions{
- PageSize: 10,
- Page: 0,
- },
- IncludeTags: true,
- })
- assert.NoError(t, err)
- assert.Len(t, releases, 8)
-
- releases, err = models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
- ListOptions: db.ListOptions{
- PageSize: 10,
- Page: 0,
- },
- IncludeTags: false,
- })
- assert.NoError(t, err)
- assert.Len(t, releases, 1)
-
- issues, err := models.Issues(&models.IssuesOptions{
- RepoIDs: []int64{repo.ID},
- IsPull: util.OptionalBoolFalse,
- SortType: "oldest",
- })
- assert.NoError(t, err)
- assert.Len(t, issues, 15)
- assert.NoError(t, issues[0].LoadDiscussComments())
- assert.Empty(t, issues[0].Comments)
-
- pulls, _, err := models.PullRequests(repo.ID, &models.PullRequestsOptions{
- SortType: "oldest",
- })
- assert.NoError(t, err)
- assert.Len(t, pulls, 30)
- assert.NoError(t, pulls[0].LoadIssue())
- assert.NoError(t, pulls[0].Issue.LoadDiscussComments())
- assert.Len(t, pulls[0].Issue.Comments, 2)
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "crypto/tls"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "strconv"
- "strings"
- "time"
-
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/proxy"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/structs"
- "code.gitea.io/gitea/modules/util"
-
- "github.com/google/go-github/v39/github"
- "golang.org/x/oauth2"
-)
-
-var (
- _ base.Downloader = &GithubDownloaderV3{}
- _ base.DownloaderFactory = &GithubDownloaderV3Factory{}
- // GithubLimitRateRemaining limit to wait for new rate to apply
- GithubLimitRateRemaining = 0
-)
-
-func init() {
- RegisterDownloaderFactory(&GithubDownloaderV3Factory{})
-}
-
-// GithubDownloaderV3Factory defines a github downloader v3 factory
-type GithubDownloaderV3Factory struct {
-}
-
-// New returns a Downloader related to this factory according MigrateOptions
-func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
- u, err := url.Parse(opts.CloneAddr)
- if err != nil {
- return nil, err
- }
-
- baseURL := u.Scheme + "://" + u.Host
- fields := strings.Split(u.Path, "/")
- oldOwner := fields[1]
- oldName := strings.TrimSuffix(fields[2], ".git")
-
- log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
-
- return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
-}
-
-// GitServiceType returns the type of git service
-func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
- return structs.GithubService
-}
-
-// GithubDownloaderV3 implements a Downloader interface to get repository information
-// from github via APIv3
-type GithubDownloaderV3 struct {
- base.NullDownloader
- ctx context.Context
- clients []*github.Client
- repoOwner string
- repoName string
- userName string
- password string
- rates []*github.Rate
- curClientIdx int
- maxPerPage int
- SkipReactions bool
-}
-
-// NewGithubDownloaderV3 creates a github Downloader via github v3 API
-func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
- var downloader = GithubDownloaderV3{
- userName: userName,
- password: password,
- ctx: ctx,
- repoOwner: repoOwner,
- repoName: repoName,
- maxPerPage: 100,
- }
-
- if token != "" {
- tokens := strings.Split(token, ",")
- for _, token := range tokens {
- token = strings.TrimSpace(token)
- ts := oauth2.StaticTokenSource(
- &oauth2.Token{AccessToken: token},
- )
- var client = &http.Client{
- Transport: &oauth2.Transport{
- Base: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
- Proxy: func(req *http.Request) (*url.URL, error) {
- return proxy.Proxy()(req)
- },
- },
- Source: oauth2.ReuseTokenSource(nil, ts),
- },
- }
-
- downloader.addClient(client, baseURL)
- }
- } else {
- var client = &http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
- Proxy: func(req *http.Request) (*url.URL, error) {
- req.SetBasicAuth(userName, password)
- return proxy.Proxy()(req)
- },
- },
- }
- downloader.addClient(client, baseURL)
- }
- return &downloader
-}
-
-func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {
- githubClient := github.NewClient(client)
- if baseURL != "https://github.com" {
- githubClient, _ = github.NewEnterpriseClient(baseURL, baseURL, client)
- }
- g.clients = append(g.clients, githubClient)
- g.rates = append(g.rates, nil)
-}
-
-// SetContext set context
-func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
- g.ctx = ctx
-}
-
-func (g *GithubDownloaderV3) waitAndPickClient() {
- var recentIdx int
- var maxRemaining int
- for i := 0; i < len(g.clients); i++ {
- if g.rates[i] != nil && g.rates[i].Remaining > maxRemaining {
- maxRemaining = g.rates[i].Remaining
- recentIdx = i
- }
- }
- g.curClientIdx = recentIdx // if no max remain, it will always pick the first client.
-
- for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {
- timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))
- select {
- case <-g.ctx.Done():
- util.StopTimer(timer)
- return
- case <-timer.C:
- }
-
- err := g.RefreshRate()
- if err != nil {
- log.Error("g.getClient().RateLimits: %s", err)
- }
- }
-}
-
-// RefreshRate update the current rate (doesn't count in rate limit)
-func (g *GithubDownloaderV3) RefreshRate() error {
- rates, _, err := g.getClient().RateLimits(g.ctx)
- if err != nil {
- // if rate limit is not enabled, ignore it
- if strings.Contains(err.Error(), "404") {
- g.setRate(nil)
- return nil
- }
- return err
- }
-
- g.setRate(rates.GetCore())
- return nil
-}
-
-func (g *GithubDownloaderV3) getClient() *github.Client {
- return g.clients[g.curClientIdx]
-}
-
-func (g *GithubDownloaderV3) setRate(rate *github.Rate) {
- g.rates[g.curClientIdx] = rate
-}
-
-// GetRepoInfo returns a repository information
-func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
- g.waitAndPickClient()
- gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
- if err != nil {
- return nil, err
- }
- g.setRate(&resp.Rate)
-
- // convert github repo to stand Repo
- return &base.Repository{
- Owner: g.repoOwner,
- Name: gr.GetName(),
- IsPrivate: gr.GetPrivate(),
- Description: gr.GetDescription(),
- OriginalURL: gr.GetHTMLURL(),
- CloneURL: gr.GetCloneURL(),
- DefaultBranch: gr.GetDefaultBranch(),
- }, nil
-}
-
-// GetTopics return github topics
-func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
- g.waitAndPickClient()
- r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
- if err != nil {
- return nil, err
- }
- g.setRate(&resp.Rate)
- return r.Topics, nil
-}
-
-// GetMilestones returns milestones
-func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
- var perPage = g.maxPerPage
- var milestones = make([]*base.Milestone, 0, perPage)
- for i := 1; ; i++ {
- g.waitAndPickClient()
- ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
- &github.MilestoneListOptions{
- State: "all",
- ListOptions: github.ListOptions{
- Page: i,
- PerPage: perPage,
- }})
- if err != nil {
- return nil, err
- }
- g.setRate(&resp.Rate)
-
- for _, m := range ms {
- var state = "open"
- if m.State != nil {
- state = *m.State
- }
- milestones = append(milestones, &base.Milestone{
- Title: m.GetTitle(),
- Description: m.GetDescription(),
- Deadline: m.DueOn,
- State: state,
- Created: m.GetCreatedAt(),
- Updated: m.UpdatedAt,
- Closed: m.ClosedAt,
- })
- }
- if len(ms) < perPage {
- break
- }
- }
- return milestones, nil
-}
-
-func convertGithubLabel(label *github.Label) *base.Label {
- return &base.Label{
- Name: label.GetName(),
- Color: label.GetColor(),
- Description: label.GetDescription(),
- }
-}
-
-// GetLabels returns labels
-func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
- var perPage = g.maxPerPage
- var labels = make([]*base.Label, 0, perPage)
- for i := 1; ; i++ {
- g.waitAndPickClient()
- ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
- &github.ListOptions{
- Page: i,
- PerPage: perPage,
- })
- if err != nil {
- return nil, err
- }
- g.setRate(&resp.Rate)
-
- for _, label := range ls {
- labels = append(labels, convertGithubLabel(label))
- }
- if len(ls) < perPage {
- break
- }
- }
- return labels, nil
-}
-
-func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release {
- r := &base.Release{
- Name: rel.GetName(),
- TagName: rel.GetTagName(),
- TargetCommitish: rel.GetTargetCommitish(),
- Draft: rel.GetDraft(),
- Prerelease: rel.GetPrerelease(),
- Created: rel.GetCreatedAt().Time,
- PublisherID: rel.GetAuthor().GetID(),
- PublisherName: rel.GetAuthor().GetLogin(),
- PublisherEmail: rel.GetAuthor().GetEmail(),
- Body: rel.GetBody(),
- }
-
- if rel.PublishedAt != nil {
- r.Published = rel.PublishedAt.Time
- }
-
- httpClient := &http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
- Proxy: proxy.Proxy(),
- },
- }
-
- for _, asset := range rel.Assets {
- var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
- r.Assets = append(r.Assets, &base.ReleaseAsset{
- ID: asset.GetID(),
- Name: asset.GetName(),
- ContentType: asset.ContentType,
- Size: asset.Size,
- DownloadCount: asset.DownloadCount,
- Created: asset.CreatedAt.Time,
- Updated: asset.UpdatedAt.Time,
- DownloadFunc: func() (io.ReadCloser, error) {
- g.waitAndPickClient()
- asset, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
- if err != nil {
- return nil, err
- }
- if err := g.RefreshRate(); err != nil {
- log.Error("g.getClient().RateLimits: %s", err)
- }
- if asset == nil {
- if redirectURL != "" {
- g.waitAndPickClient()
- req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
- if err != nil {
- return nil, err
- }
- resp, err := httpClient.Do(req)
- err1 := g.RefreshRate()
- if err1 != nil {
- log.Error("g.getClient().RateLimits: %s", err1)
- }
- if err != nil {
- return nil, err
- }
- return resp.Body, nil
- }
- return nil, fmt.Errorf("No release asset found for %d", assetID)
- }
- return asset, nil
- },
- })
- }
- return r
-}
-
-// GetReleases returns releases
-func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
- var perPage = g.maxPerPage
- var releases = make([]*base.Release, 0, perPage)
- for i := 1; ; i++ {
- g.waitAndPickClient()
- ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
- &github.ListOptions{
- Page: i,
- PerPage: perPage,
- })
- if err != nil {
- return nil, err
- }
- g.setRate(&resp.Rate)
-
- for _, release := range ls {
- releases = append(releases, g.convertGithubRelease(release))
- }
- if len(ls) < perPage {
- break
- }
- }
- return releases, nil
-}
-
-// GetIssues returns issues according start and limit
-func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
- opt := &github.IssueListByRepoOptions{
- Sort: "created",
- Direction: "asc",
- State: "all",
- ListOptions: github.ListOptions{
- PerPage: perPage,
- Page: page,
- },
- }
-
- var allIssues = make([]*base.Issue, 0, perPage)
- g.waitAndPickClient()
- issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
- if err != nil {
- return nil, false, fmt.Errorf("error while listing repos: %v", err)
- }
- log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
- g.setRate(&resp.Rate)
- for _, issue := range issues {
- if issue.IsPullRequest() {
- continue
- }
-
- var labels = make([]*base.Label, 0, len(issue.Labels))
- for _, l := range issue.Labels {
- labels = append(labels, convertGithubLabel(l))
- }
-
- // get reactions
- var reactions []*base.Reaction
- if !g.SkipReactions {
- for i := 1; ; i++ {
- g.waitAndPickClient()
- res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
- Page: i,
- PerPage: perPage,
- })
- if err != nil {
- return nil, false, err
- }
- g.setRate(&resp.Rate)
- if len(res) == 0 {
- break
- }
- for _, reaction := range res {
- reactions = append(reactions, &base.Reaction{
- UserID: reaction.User.GetID(),
- UserName: reaction.User.GetLogin(),
- Content: reaction.GetContent(),
- })
- }
- }
- }
-
- var assignees []string
- for i := range issue.Assignees {
- assignees = append(assignees, issue.Assignees[i].GetLogin())
- }
-
- allIssues = append(allIssues, &base.Issue{
- Title: *issue.Title,
- Number: int64(*issue.Number),
- PosterID: issue.GetUser().GetID(),
- PosterName: issue.GetUser().GetLogin(),
- PosterEmail: issue.GetUser().GetEmail(),
- Content: issue.GetBody(),
- Milestone: issue.GetMilestone().GetTitle(),
- State: issue.GetState(),
- Created: issue.GetCreatedAt(),
- Updated: issue.GetUpdatedAt(),
- Labels: labels,
- Reactions: reactions,
- Closed: issue.ClosedAt,
- IsLocked: issue.GetLocked(),
- Assignees: assignees,
- Context: base.BasicIssueContext(*issue.Number),
- })
- }
-
- return allIssues, len(issues) < perPage, nil
-}
-
-// SupportGetRepoComments return true if it supports get repo comments
-func (g *GithubDownloaderV3) SupportGetRepoComments() bool {
- return true
-}
-
-// GetComments returns comments according issueNumber
-func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
- if opts.Context != nil {
- comments, err := g.getComments(opts.Context)
- return comments, false, err
- }
-
- return g.GetAllComments(opts.Page, opts.PageSize)
-}
-
-func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*base.Comment, error) {
- var (
- allComments = make([]*base.Comment, 0, g.maxPerPage)
- created = "created"
- asc = "asc"
- )
- opt := &github.IssueListCommentsOptions{
- Sort: &created,
- Direction: &asc,
- ListOptions: github.ListOptions{
- PerPage: g.maxPerPage,
- },
- }
- for {
- g.waitAndPickClient()
- comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt)
- if err != nil {
- return nil, fmt.Errorf("error while listing repos: %v", err)
- }
- g.setRate(&resp.Rate)
- for _, comment := range comments {
- // get reactions
- var reactions []*base.Reaction
- if !g.SkipReactions {
- for i := 1; ; i++ {
- g.waitAndPickClient()
- res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
- Page: i,
- PerPage: g.maxPerPage,
- })
- if err != nil {
- return nil, err
- }
- g.setRate(&resp.Rate)
- if len(res) == 0 {
- break
- }
- for _, reaction := range res {
- reactions = append(reactions, &base.Reaction{
- UserID: reaction.User.GetID(),
- UserName: reaction.User.GetLogin(),
- Content: reaction.GetContent(),
- })
- }
- }
- }
-
- allComments = append(allComments, &base.Comment{
- IssueIndex: issueContext.LocalID(),
- PosterID: comment.GetUser().GetID(),
- PosterName: comment.GetUser().GetLogin(),
- PosterEmail: comment.GetUser().GetEmail(),
- Content: comment.GetBody(),
- Created: comment.GetCreatedAt(),
- Updated: comment.GetUpdatedAt(),
- Reactions: reactions,
- })
- }
- if resp.NextPage == 0 {
- break
- }
- opt.Page = resp.NextPage
- }
- return allComments, nil
-}
-
-// GetAllComments returns repository comments according page and perPageSize
-func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) {
- var (
- allComments = make([]*base.Comment, 0, perPage)
- created = "created"
- asc = "asc"
- )
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
- opt := &github.IssueListCommentsOptions{
- Sort: &created,
- Direction: &asc,
- ListOptions: github.ListOptions{
- Page: page,
- PerPage: perPage,
- },
- }
-
- g.waitAndPickClient()
- comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt)
- if err != nil {
- return nil, false, fmt.Errorf("error while listing repos: %v", err)
- }
- var isEnd = resp.NextPage == 0
-
- log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage)
- g.setRate(&resp.Rate)
- for _, comment := range comments {
- // get reactions
- var reactions []*base.Reaction
- if !g.SkipReactions {
- for i := 1; ; i++ {
- g.waitAndPickClient()
- res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
- Page: i,
- PerPage: g.maxPerPage,
- })
- if err != nil {
- return nil, false, err
- }
- g.setRate(&resp.Rate)
- if len(res) == 0 {
- break
- }
- for _, reaction := range res {
- reactions = append(reactions, &base.Reaction{
- UserID: reaction.User.GetID(),
- UserName: reaction.User.GetLogin(),
- Content: reaction.GetContent(),
- })
- }
- }
- }
- idx := strings.LastIndex(*comment.IssueURL, "/")
- issueIndex, _ := strconv.ParseInt((*comment.IssueURL)[idx+1:], 10, 64)
- allComments = append(allComments, &base.Comment{
- IssueIndex: issueIndex,
- PosterID: comment.GetUser().GetID(),
- PosterName: comment.GetUser().GetLogin(),
- PosterEmail: comment.GetUser().GetEmail(),
- Content: comment.GetBody(),
- Created: comment.GetCreatedAt(),
- Updated: comment.GetUpdatedAt(),
- Reactions: reactions,
- })
- }
-
- return allComments, isEnd, nil
-}
-
-// GetPullRequests returns pull requests according page and perPage
-func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
- opt := &github.PullRequestListOptions{
- Sort: "created",
- Direction: "asc",
- State: "all",
- ListOptions: github.ListOptions{
- PerPage: perPage,
- Page: page,
- },
- }
- var allPRs = make([]*base.PullRequest, 0, perPage)
- g.waitAndPickClient()
- prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
- if err != nil {
- return nil, false, fmt.Errorf("error while listing repos: %v", err)
- }
- log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
- g.setRate(&resp.Rate)
- for _, pr := range prs {
- var labels = make([]*base.Label, 0, len(pr.Labels))
- for _, l := range pr.Labels {
- labels = append(labels, convertGithubLabel(l))
- }
-
- // get reactions
- var reactions []*base.Reaction
- if !g.SkipReactions {
- for i := 1; ; i++ {
- g.waitAndPickClient()
- res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
- Page: i,
- PerPage: perPage,
- })
- if err != nil {
- return nil, false, err
- }
- g.setRate(&resp.Rate)
- if len(res) == 0 {
- break
- }
- for _, reaction := range res {
- reactions = append(reactions, &base.Reaction{
- UserID: reaction.User.GetID(),
- UserName: reaction.User.GetLogin(),
- Content: reaction.GetContent(),
- })
- }
- }
- }
-
- // download patch and saved as tmp file
- g.waitAndPickClient()
-
- allPRs = append(allPRs, &base.PullRequest{
- Title: pr.GetTitle(),
- Number: int64(pr.GetNumber()),
- PosterID: pr.GetUser().GetID(),
- PosterName: pr.GetUser().GetLogin(),
- PosterEmail: pr.GetUser().GetEmail(),
- Content: pr.GetBody(),
- Milestone: pr.GetMilestone().GetTitle(),
- State: pr.GetState(),
- Created: pr.GetCreatedAt(),
- Updated: pr.GetUpdatedAt(),
- Closed: pr.ClosedAt,
- Labels: labels,
- Merged: pr.MergedAt != nil,
- MergeCommitSHA: pr.GetMergeCommitSHA(),
- MergedTime: pr.MergedAt,
- IsLocked: pr.ActiveLockReason != nil,
- Head: base.PullRequestBranch{
- Ref: pr.GetHead().GetRef(),
- SHA: pr.GetHead().GetSHA(),
- OwnerName: pr.GetHead().GetUser().GetLogin(),
- RepoName: pr.GetHead().GetRepo().GetName(),
- CloneURL: pr.GetHead().GetRepo().GetCloneURL(),
- },
- Base: base.PullRequestBranch{
- Ref: pr.GetBase().GetRef(),
- SHA: pr.GetBase().GetSHA(),
- RepoName: pr.GetBase().GetRepo().GetName(),
- OwnerName: pr.GetBase().GetUser().GetLogin(),
- },
- PatchURL: pr.GetPatchURL(),
- Reactions: reactions,
- Context: base.BasicIssueContext(*pr.Number),
- })
- }
-
- return allPRs, len(prs) < perPage, nil
-}
-
-func convertGithubReview(r *github.PullRequestReview) *base.Review {
- return &base.Review{
- ID: r.GetID(),
- ReviewerID: r.GetUser().GetID(),
- ReviewerName: r.GetUser().GetLogin(),
- CommitID: r.GetCommitID(),
- Content: r.GetBody(),
- CreatedAt: r.GetSubmittedAt(),
- State: r.GetState(),
- }
-}
-
-func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
- var rcs = make([]*base.ReviewComment, 0, len(cs))
- for _, c := range cs {
- // get reactions
- var reactions []*base.Reaction
- if !g.SkipReactions {
- for i := 1; ; i++ {
- g.waitAndPickClient()
- res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
- Page: i,
- PerPage: g.maxPerPage,
- })
- if err != nil {
- return nil, err
- }
- g.setRate(&resp.Rate)
- if len(res) == 0 {
- break
- }
- for _, reaction := range res {
- reactions = append(reactions, &base.Reaction{
- UserID: reaction.User.GetID(),
- UserName: reaction.User.GetLogin(),
- Content: reaction.GetContent(),
- })
- }
- }
- }
-
- rcs = append(rcs, &base.ReviewComment{
- ID: c.GetID(),
- InReplyTo: c.GetInReplyTo(),
- Content: c.GetBody(),
- TreePath: c.GetPath(),
- DiffHunk: c.GetDiffHunk(),
- Position: c.GetPosition(),
- CommitID: c.GetCommitID(),
- PosterID: c.GetUser().GetID(),
- Reactions: reactions,
- CreatedAt: c.GetCreatedAt(),
- UpdatedAt: c.GetUpdatedAt(),
- })
- }
- return rcs, nil
-}
-
-// GetReviews returns pull requests review
-func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Review, error) {
- var allReviews = make([]*base.Review, 0, g.maxPerPage)
- opt := &github.ListOptions{
- PerPage: g.maxPerPage,
- }
- for {
- g.waitAndPickClient()
- reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt)
- if err != nil {
- return nil, fmt.Errorf("error while listing repos: %v", err)
- }
- g.setRate(&resp.Rate)
- for _, review := range reviews {
- r := convertGithubReview(review)
- r.IssueIndex = context.LocalID()
- // retrieve all review comments
- opt2 := &github.ListOptions{
- PerPage: g.maxPerPage,
- }
- for {
- g.waitAndPickClient()
- reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2)
- if err != nil {
- return nil, fmt.Errorf("error while listing repos: %v", err)
- }
- g.setRate(&resp.Rate)
-
- cs, err := g.convertGithubReviewComments(reviewComments)
- if err != nil {
- return nil, err
- }
- r.Comments = append(r.Comments, cs...)
- if resp.NextPage == 0 {
- break
- }
- opt2.Page = resp.NextPage
- }
- allReviews = append(allReviews, r)
- }
- if resp.NextPage == 0 {
- break
- }
- opt.Page = resp.NextPage
- }
- return allReviews, nil
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "os"
- "testing"
- "time"
-
- "code.gitea.io/gitea/modules/migrations/base"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestGitHubDownloadRepo(t *testing.T) {
- GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in //
- downloader := NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", os.Getenv("GITHUB_READ_TOKEN"), "go-gitea", "test_repo")
- err := downloader.RefreshRate()
- assert.NoError(t, err)
-
- repo, err := downloader.GetRepoInfo()
- assert.NoError(t, err)
- assertRepositoryEqual(t, &base.Repository{
- Name: "test_repo",
- Owner: "go-gitea",
- Description: "Test repository for testing migration from github to gitea",
- CloneURL: "https://github.com/go-gitea/test_repo.git",
- OriginalURL: "https://github.com/go-gitea/test_repo",
- DefaultBranch: "master",
- }, repo)
-
- topics, err := downloader.GetTopics()
- assert.NoError(t, err)
- assert.Contains(t, topics, "gitea")
-
- milestones, err := downloader.GetMilestones()
- assert.NoError(t, err)
- assertMilestonesEqual(t, []*base.Milestone{
- {
- Title: "1.0.0",
- Description: "Milestone 1.0.0",
- Deadline: timePtr(time.Date(2019, 11, 11, 8, 0, 0, 0, time.UTC)),
- Created: time.Date(2019, 11, 12, 19, 37, 8, 0, time.UTC),
- Updated: timePtr(time.Date(2019, 11, 12, 21, 56, 17, 0, time.UTC)),
- Closed: timePtr(time.Date(2019, 11, 12, 19, 45, 49, 0, time.UTC)),
- State: "closed",
- },
- {
- Title: "1.1.0",
- Description: "Milestone 1.1.0",
- Deadline: timePtr(time.Date(2019, 11, 12, 8, 0, 0, 0, time.UTC)),
- Created: time.Date(2019, 11, 12, 19, 37, 25, 0, time.UTC),
- Updated: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
- Closed: timePtr(time.Date(2019, 11, 12, 19, 45, 46, 0, time.UTC)),
- State: "closed",
- },
- }, milestones)
-
- labels, err := downloader.GetLabels()
- assert.NoError(t, err)
- assertLabelsEqual(t, []*base.Label{
- {
- Name: "bug",
- Color: "d73a4a",
- Description: "Something isn't working",
- },
- {
- Name: "documentation",
- Color: "0075ca",
- Description: "Improvements or additions to documentation",
- },
- {
- Name: "duplicate",
- Color: "cfd3d7",
- Description: "This issue or pull request already exists",
- },
- {
- Name: "enhancement",
- Color: "a2eeef",
- Description: "New feature or request",
- },
- {
- Name: "good first issue",
- Color: "7057ff",
- Description: "Good for newcomers",
- },
- {
- Name: "help wanted",
- Color: "008672",
- Description: "Extra attention is needed",
- },
- {
- Name: "invalid",
- Color: "e4e669",
- Description: "This doesn't seem right",
- },
- {
- Name: "question",
- Color: "d876e3",
- Description: "Further information is requested",
- },
- {
- Name: "wontfix",
- Color: "ffffff",
- Description: "This will not be worked on",
- },
- }, labels)
-
- releases, err := downloader.GetReleases()
- assert.NoError(t, err)
- assertReleasesEqual(t, []*base.Release{
- {
- TagName: "v0.9.99",
- TargetCommitish: "master",
- Name: "First Release",
- Body: "A test release",
- Created: time.Date(2019, 11, 9, 16, 49, 21, 0, time.UTC),
- Published: time.Date(2019, 11, 12, 20, 12, 10, 0, time.UTC),
- PublisherID: 1669571,
- PublisherName: "mrsdizzie",
- },
- }, releases)
-
- // downloader.GetIssues()
- issues, isEnd, err := downloader.GetIssues(1, 2)
- assert.NoError(t, err)
- assert.False(t, isEnd)
- assertIssuesEqual(t, []*base.Issue{
- {
- Number: 1,
- Title: "Please add an animated gif icon to the merge button",
- Content: "I just want the merge button to hurt my eyes a little. \xF0\x9F\x98\x9D ",
- Milestone: "1.0.0",
- PosterID: 18600385,
- PosterName: "guillep2k",
- State: "closed",
- Created: time.Date(2019, 11, 9, 17, 0, 29, 0, time.UTC),
- Updated: time.Date(2019, 11, 12, 20, 29, 53, 0, time.UTC),
- Labels: []*base.Label{
- {
- Name: "bug",
- Color: "d73a4a",
- Description: "Something isn't working",
- },
- {
- Name: "good first issue",
- Color: "7057ff",
- Description: "Good for newcomers",
- },
- },
- Reactions: []*base.Reaction{
- {
- UserID: 1669571,
- UserName: "mrsdizzie",
- Content: "+1",
- },
- },
- Closed: timePtr(time.Date(2019, 11, 12, 20, 22, 22, 0, time.UTC)),
- },
- {
- Number: 2,
- Title: "Test issue",
- Content: "This is test issue 2, do not touch!",
- Milestone: "1.1.0",
- PosterID: 1669571,
- PosterName: "mrsdizzie",
- State: "closed",
- Created: time.Date(2019, 11, 12, 21, 0, 6, 0, time.UTC),
- Updated: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
- Labels: []*base.Label{
- {
- Name: "duplicate",
- Color: "cfd3d7",
- Description: "This issue or pull request already exists",
- },
- },
- Reactions: []*base.Reaction{
- {
- UserID: 1669571,
- UserName: "mrsdizzie",
- Content: "heart",
- },
- {
- UserID: 1669571,
- UserName: "mrsdizzie",
- Content: "laugh",
- },
- {
- UserID: 1669571,
- UserName: "mrsdizzie",
- Content: "-1",
- },
- {
- UserID: 1669571,
- UserName: "mrsdizzie",
- Content: "confused",
- },
- {
- UserID: 1669571,
- UserName: "mrsdizzie",
- Content: "hooray",
- },
- {
- UserID: 1669571,
- UserName: "mrsdizzie",
- Content: "+1",
- },
- },
- Closed: timePtr(time.Date(2019, 11, 12, 21, 1, 31, 0, time.UTC)),
- },
- }, issues)
-
- // downloader.GetComments()
- comments, _, err := downloader.GetComments(base.GetCommentOptions{
- Context: base.BasicIssueContext(2),
- })
- assert.NoError(t, err)
- assertCommentsEqual(t, []*base.Comment{
- {
- IssueIndex: 2,
- PosterID: 1669571,
- PosterName: "mrsdizzie",
- Created: time.Date(2019, 11, 12, 21, 0, 13, 0, time.UTC),
- Updated: time.Date(2019, 11, 12, 21, 0, 13, 0, time.UTC),
- Content: "This is a comment",
- Reactions: []*base.Reaction{
- {
- UserID: 1669571,
- UserName: "mrsdizzie",
- Content: "+1",
- },
- },
- },
- {
- IssueIndex: 2,
- PosterID: 1669571,
- PosterName: "mrsdizzie",
- Created: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
- Updated: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
- Content: "A second comment",
- Reactions: nil,
- },
- }, comments)
-
- // downloader.GetPullRequests()
- prs, _, err := downloader.GetPullRequests(1, 2)
- assert.NoError(t, err)
- assertPullRequestsEqual(t, []*base.PullRequest{
- {
- Number: 3,
- Title: "Update README.md",
- Content: "add warning to readme",
- Milestone: "1.1.0",
- PosterID: 1669571,
- PosterName: "mrsdizzie",
- State: "closed",
- Created: time.Date(2019, 11, 12, 21, 21, 43, 0, time.UTC),
- Updated: time.Date(2019, 11, 12, 21, 39, 28, 0, time.UTC),
- Labels: []*base.Label{
- {
- Name: "documentation",
- Color: "0075ca",
- Description: "Improvements or additions to documentation",
- },
- },
- PatchURL: "https://github.com/go-gitea/test_repo/pull/3.patch",
- Head: base.PullRequestBranch{
- Ref: "master",
- CloneURL: "https://github.com/mrsdizzie/test_repo.git",
- SHA: "076160cf0b039f13e5eff19619932d181269414b",
- RepoName: "test_repo",
-
- OwnerName: "mrsdizzie",
- },
- Base: base.PullRequestBranch{
- Ref: "master",
- SHA: "72866af952e98d02a73003501836074b286a78f6",
- OwnerName: "go-gitea",
- RepoName: "test_repo",
- },
- Closed: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
- Merged: true,
- MergedTime: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
- MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
- Context: base.BasicIssueContext(3),
- },
- {
- Number: 4,
- Title: "Test branch",
- Content: "do not merge this PR",
- Milestone: "1.0.0",
- PosterID: 1669571,
- PosterName: "mrsdizzie",
- State: "open",
- Created: time.Date(2019, 11, 12, 21, 54, 18, 0, time.UTC),
- Updated: time.Date(2020, 1, 4, 11, 30, 1, 0, time.UTC),
- Labels: []*base.Label{
- {
- Name: "bug",
- Color: "d73a4a",
- Description: "Something isn't working",
- },
- },
- PatchURL: "https://github.com/go-gitea/test_repo/pull/4.patch",
- Head: base.PullRequestBranch{
- Ref: "test-branch",
- SHA: "2be9101c543658591222acbee3eb799edfc3853d",
- RepoName: "test_repo",
- OwnerName: "mrsdizzie",
- CloneURL: "https://github.com/mrsdizzie/test_repo.git",
- },
- Base: base.PullRequestBranch{
- Ref: "master",
- SHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
- OwnerName: "go-gitea",
- RepoName: "test_repo",
- },
- Merged: false,
- MergeCommitSHA: "565d1208f5fffdc1c5ae1a2436491eb9a5e4ebae",
- Reactions: []*base.Reaction{
- {
- UserID: 81045,
- UserName: "lunny",
- Content: "heart",
- },
- {
- UserID: 81045,
- UserName: "lunny",
- Content: "+1",
- },
- },
- Context: base.BasicIssueContext(4),
- },
- }, prs)
-
- reviews, err := downloader.GetReviews(base.BasicIssueContext(3))
- assert.NoError(t, err)
- assertReviewsEqual(t, []*base.Review{
- {
- ID: 315859956,
- IssueIndex: 3,
- ReviewerID: 42128690,
- ReviewerName: "jolheiser",
- CommitID: "076160cf0b039f13e5eff19619932d181269414b",
- CreatedAt: time.Date(2019, 11, 12, 21, 35, 24, 0, time.UTC),
- State: base.ReviewStateApproved,
- },
- {
- ID: 315860062,
- IssueIndex: 3,
- ReviewerID: 1824502,
- ReviewerName: "zeripath",
- CommitID: "076160cf0b039f13e5eff19619932d181269414b",
- CreatedAt: time.Date(2019, 11, 12, 21, 35, 36, 0, time.UTC),
- State: base.ReviewStateApproved,
- },
- {
- ID: 315861440,
- IssueIndex: 3,
- ReviewerID: 165205,
- ReviewerName: "lafriks",
- CommitID: "076160cf0b039f13e5eff19619932d181269414b",
- CreatedAt: time.Date(2019, 11, 12, 21, 38, 00, 0, time.UTC),
- State: base.ReviewStateApproved,
- },
- }, reviews)
-
- reviews, err = downloader.GetReviews(base.BasicIssueContext(4))
- assert.NoError(t, err)
- assertReviewsEqual(t, []*base.Review{
- {
- ID: 338338740,
- IssueIndex: 4,
- ReviewerID: 81045,
- ReviewerName: "lunny",
- CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
- CreatedAt: time.Date(2020, 01, 04, 05, 33, 18, 0, time.UTC),
- State: base.ReviewStateApproved,
- Comments: []*base.ReviewComment{
- {
- ID: 363017488,
- Content: "This is a good pull request.",
- TreePath: "README.md",
- DiffHunk: "@@ -1,2 +1,4 @@\n # test_repo\n Test repository for testing migration from github to gitea\n+",
- Position: 3,
- CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
- PosterID: 81045,
- CreatedAt: time.Date(2020, 01, 04, 05, 33, 06, 0, time.UTC),
- UpdatedAt: time.Date(2020, 01, 04, 05, 33, 18, 0, time.UTC),
- },
- },
- },
- {
- ID: 338339651,
- IssueIndex: 4,
- ReviewerID: 81045,
- ReviewerName: "lunny",
- CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
- CreatedAt: time.Date(2020, 01, 04, 06, 07, 06, 0, time.UTC),
- State: base.ReviewStateChangesRequested,
- Content: "Don't add more reviews",
- },
- {
- ID: 338349019,
- IssueIndex: 4,
- ReviewerID: 81045,
- ReviewerName: "lunny",
- CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
- CreatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
- State: base.ReviewStateCommented,
- Comments: []*base.ReviewComment{
- {
- ID: 363029944,
- Content: "test a single comment.",
- TreePath: "LICENSE",
- DiffHunk: "@@ -19,3 +19,5 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n SOFTWARE.\n+",
- Position: 4,
- CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
- PosterID: 81045,
- CreatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
- UpdatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
- },
- },
- },
- }, reviews)
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "crypto/tls"
- "errors"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "path"
- "strings"
- "time"
-
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/proxy"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/structs"
-
- "github.com/xanzy/go-gitlab"
-)
-
-var (
- _ base.Downloader = &GitlabDownloader{}
- _ base.DownloaderFactory = &GitlabDownloaderFactory{}
-)
-
-func init() {
- RegisterDownloaderFactory(&GitlabDownloaderFactory{})
-}
-
-// GitlabDownloaderFactory defines a gitlab downloader factory
-type GitlabDownloaderFactory struct {
-}
-
-// New returns a Downloader related to this factory according MigrateOptions
-func (f *GitlabDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
- u, err := url.Parse(opts.CloneAddr)
- if err != nil {
- return nil, err
- }
-
- baseURL := u.Scheme + "://" + u.Host
- repoNameSpace := strings.TrimPrefix(u.Path, "/")
- repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
-
- log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
-
- return NewGitlabDownloader(ctx, baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken)
-}
-
-// GitServiceType returns the type of git service
-func (f *GitlabDownloaderFactory) GitServiceType() structs.GitServiceType {
- return structs.GitlabService
-}
-
-// GitlabDownloader implements a Downloader interface to get repository information
-// from gitlab via go-gitlab
-// - issueCount is incremented in GetIssues() to ensure PR and Issue numbers do not overlap,
-// because Gitlab has individual Issue and Pull Request numbers.
-type GitlabDownloader struct {
- base.NullDownloader
- ctx context.Context
- client *gitlab.Client
- repoID int
- repoName string
- issueCount int64
- maxPerPage int
-}
-
-// NewGitlabDownloader creates a gitlab Downloader via gitlab API
-// Use either a username/password, personal token entered into the username field, or anonymous/public access
-// Note: Public access only allows very basic access
-func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
- gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
- Proxy: proxy.Proxy(),
- },
- }))
- // Only use basic auth if token is blank and password is NOT
- // Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
- if token == "" && password != "" {
- gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
- }
-
- if err != nil {
- log.Trace("Error logging into gitlab: %v", err)
- return nil, err
- }
-
- // split namespace and subdirectory
- pathParts := strings.Split(strings.Trim(repoPath, "/"), "/")
- var resp *gitlab.Response
- u, _ := url.Parse(baseURL)
- for len(pathParts) >= 2 {
- _, resp, err = gitlabClient.Version.GetVersion()
- if err == nil || resp != nil && resp.StatusCode == 401 {
- err = nil // if no authentication given, this still should work
- break
- }
-
- u.Path = path.Join(u.Path, pathParts[0])
- baseURL = u.String()
- pathParts = pathParts[1:]
- _ = gitlab.WithBaseURL(baseURL)(gitlabClient)
- repoPath = strings.Join(pathParts, "/")
- }
- if err != nil {
- log.Trace("Error could not get gitlab version: %v", err)
- return nil, err
- }
-
- log.Trace("gitlab downloader: use BaseURL: '%s' and RepoPath: '%s'", baseURL, repoPath)
-
- // Grab and store project/repo ID here, due to issues using the URL escaped path
- gr, _, err := gitlabClient.Projects.GetProject(repoPath, nil, nil, gitlab.WithContext(ctx))
- if err != nil {
- log.Trace("Error retrieving project: %v", err)
- return nil, err
- }
-
- if gr == nil {
- log.Trace("Error getting project, project is nil")
- return nil, errors.New("Error getting project, project is nil")
- }
-
- return &GitlabDownloader{
- ctx: ctx,
- client: gitlabClient,
- repoID: gr.ID,
- repoName: gr.Name,
- maxPerPage: 100,
- }, nil
-}
-
-// SetContext set context
-func (g *GitlabDownloader) SetContext(ctx context.Context) {
- g.ctx = ctx
-}
-
-// GetRepoInfo returns a repository information
-func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) {
- gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
-
- var private bool
- switch gr.Visibility {
- case gitlab.InternalVisibility:
- private = true
- case gitlab.PrivateVisibility:
- private = true
- }
-
- var owner string
- if gr.Owner == nil {
- log.Trace("gr.Owner is nil, trying to get owner from Namespace")
- if gr.Namespace != nil && gr.Namespace.Kind == "user" {
- owner = gr.Namespace.Path
- }
- } else {
- owner = gr.Owner.Username
- }
-
- // convert gitlab repo to stand Repo
- return &base.Repository{
- Owner: owner,
- Name: gr.Name,
- IsPrivate: private,
- Description: gr.Description,
- OriginalURL: gr.WebURL,
- CloneURL: gr.HTTPURLToRepo,
- DefaultBranch: gr.DefaultBranch,
- }, nil
-}
-
-// GetTopics return gitlab topics
-func (g *GitlabDownloader) GetTopics() ([]string, error) {
- gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
- return gr.TagList, err
-}
-
-// GetMilestones returns milestones
-func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
- var perPage = g.maxPerPage
- var state = "all"
- var milestones = make([]*base.Milestone, 0, perPage)
- for i := 1; ; i++ {
- ms, _, err := g.client.Milestones.ListMilestones(g.repoID, &gitlab.ListMilestonesOptions{
- State: &state,
- ListOptions: gitlab.ListOptions{
- Page: i,
- PerPage: perPage,
- }}, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
-
- for _, m := range ms {
- var desc string
- if m.Description != "" {
- desc = m.Description
- }
- var state = "open"
- var closedAt *time.Time
- if m.State != "" {
- state = m.State
- if state == "closed" {
- closedAt = m.UpdatedAt
- }
- }
-
- var deadline *time.Time
- if m.DueDate != nil {
- deadlineParsed, err := time.Parse("2006-01-02", m.DueDate.String())
- if err != nil {
- log.Trace("Error parsing Milestone DueDate time")
- deadline = nil
- } else {
- deadline = &deadlineParsed
- }
- }
-
- milestones = append(milestones, &base.Milestone{
- Title: m.Title,
- Description: desc,
- Deadline: deadline,
- State: state,
- Created: *m.CreatedAt,
- Updated: m.UpdatedAt,
- Closed: closedAt,
- })
- }
- if len(ms) < perPage {
- break
- }
- }
- return milestones, nil
-}
-
-func (g *GitlabDownloader) normalizeColor(val string) string {
- val = strings.TrimLeft(val, "#")
- val = strings.ToLower(val)
- if len(val) == 3 {
- c := []rune(val)
- val = fmt.Sprintf("%c%c%c%c%c%c", c[0], c[0], c[1], c[1], c[2], c[2])
- }
- if len(val) != 6 {
- return ""
- }
- return val
-}
-
-// GetLabels returns labels
-func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
- var perPage = g.maxPerPage
- var labels = make([]*base.Label, 0, perPage)
- for i := 1; ; i++ {
- ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{
- Page: i,
- PerPage: perPage,
- }}, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
- for _, label := range ls {
- baseLabel := &base.Label{
- Name: label.Name,
- Color: g.normalizeColor(label.Color),
- Description: label.Description,
- }
- labels = append(labels, baseLabel)
- }
- if len(ls) < perPage {
- break
- }
- }
- return labels, nil
-}
-
-func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release {
- var zero int
- r := &base.Release{
- TagName: rel.TagName,
- TargetCommitish: rel.Commit.ID,
- Name: rel.Name,
- Body: rel.Description,
- Created: *rel.CreatedAt,
- PublisherID: int64(rel.Author.ID),
- PublisherName: rel.Author.Username,
- }
-
- httpClient := &http.Client{
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
- Proxy: proxy.Proxy(),
- },
- }
-
- for k, asset := range rel.Assets.Links {
- r.Assets = append(r.Assets, &base.ReleaseAsset{
- ID: int64(asset.ID),
- Name: asset.Name,
- ContentType: &rel.Assets.Sources[k].Format,
- Size: &zero,
- DownloadCount: &zero,
- DownloadFunc: func() (io.ReadCloser, error) {
- link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, asset.ID, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
-
- req, err := http.NewRequest("GET", link.URL, nil)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(g.ctx)
- resp, err := httpClient.Do(req)
- if err != nil {
- return nil, err
- }
-
- // resp.Body is closed by the uploader
- return resp.Body, nil
- },
- })
- }
- return r
-}
-
-// GetReleases returns releases
-func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
- var perPage = g.maxPerPage
- var releases = make([]*base.Release, 0, perPage)
- for i := 1; ; i++ {
- ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{
- Page: i,
- PerPage: perPage,
- }, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
-
- for _, release := range ls {
- releases = append(releases, g.convertGitlabRelease(release))
- }
- if len(ls) < perPage {
- break
- }
- }
- return releases, nil
-}
-
-type gitlabIssueContext struct {
- foreignID int64
- localID int64
- IsMergeRequest bool
-}
-
-func (c gitlabIssueContext) LocalID() int64 {
- return c.localID
-}
-
-func (c gitlabIssueContext) ForeignID() int64 {
- return c.foreignID
-}
-
-// GetIssues returns issues according start and limit
-// Note: issue label description and colors are not supported by the go-gitlab library at this time
-func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
- state := "all"
- sort := "asc"
-
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
-
- opt := &gitlab.ListProjectIssuesOptions{
- State: &state,
- Sort: &sort,
- ListOptions: gitlab.ListOptions{
- PerPage: perPage,
- Page: page,
- },
- }
-
- var allIssues = make([]*base.Issue, 0, perPage)
-
- issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, false, fmt.Errorf("error while listing issues: %v", err)
- }
- for _, issue := range issues {
-
- var labels = make([]*base.Label, 0, len(issue.Labels))
- for _, l := range issue.Labels {
- labels = append(labels, &base.Label{
- Name: l,
- })
- }
-
- var milestone string
- if issue.Milestone != nil {
- milestone = issue.Milestone.Title
- }
-
- var reactions []*base.Reaction
- var awardPage = 1
- for {
- awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, false, fmt.Errorf("error while listing issue awards: %v", err)
- }
-
- for i := range awards {
- reactions = append(reactions, g.awardToReaction(awards[i]))
- }
-
- if len(awards) < perPage {
- break
- }
-
- awardPage++
- }
-
- allIssues = append(allIssues, &base.Issue{
- Title: issue.Title,
- Number: int64(issue.IID),
- PosterID: int64(issue.Author.ID),
- PosterName: issue.Author.Username,
- Content: issue.Description,
- Milestone: milestone,
- State: issue.State,
- Created: *issue.CreatedAt,
- Labels: labels,
- Reactions: reactions,
- Closed: issue.ClosedAt,
- IsLocked: issue.DiscussionLocked,
- Updated: *issue.UpdatedAt,
- Context: gitlabIssueContext{
- foreignID: int64(issue.IID),
- localID: int64(issue.IID),
- IsMergeRequest: false,
- },
- })
-
- // increment issueCount, to be used in GetPullRequests()
- g.issueCount++
- }
-
- return allIssues, len(issues) < perPage, nil
-}
-
-// GetComments returns comments according issueNumber
-// TODO: figure out how to transfer comment reactions
-func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
- context, ok := opts.Context.(gitlabIssueContext)
- if !ok {
- return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context)
- }
-
- var allComments = make([]*base.Comment, 0, g.maxPerPage)
-
- var page = 1
-
- for {
- var comments []*gitlab.Discussion
- var resp *gitlab.Response
- var err error
- if !context.IsMergeRequest {
- comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListIssueDiscussionsOptions{
- Page: page,
- PerPage: g.maxPerPage,
- }, nil, gitlab.WithContext(g.ctx))
- } else {
- comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListMergeRequestDiscussionsOptions{
- Page: page,
- PerPage: g.maxPerPage,
- }, nil, gitlab.WithContext(g.ctx))
- }
-
- if err != nil {
- return nil, false, fmt.Errorf("error while listing comments: %v %v", g.repoID, err)
- }
- for _, comment := range comments {
- // Flatten comment threads
- if !comment.IndividualNote {
- for _, note := range comment.Notes {
- allComments = append(allComments, &base.Comment{
- IssueIndex: context.LocalID(),
- PosterID: int64(note.Author.ID),
- PosterName: note.Author.Username,
- PosterEmail: note.Author.Email,
- Content: note.Body,
- Created: *note.CreatedAt,
- })
- }
- } else {
- c := comment.Notes[0]
- allComments = append(allComments, &base.Comment{
- IssueIndex: context.LocalID(),
- PosterID: int64(c.Author.ID),
- PosterName: c.Author.Username,
- PosterEmail: c.Author.Email,
- Content: c.Body,
- Created: *c.CreatedAt,
- })
- }
-
- }
- if resp.NextPage == 0 {
- break
- }
- page = resp.NextPage
- }
- return allComments, true, nil
-}
-
-// GetPullRequests returns pull requests according page and perPage
-func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
- if perPage > g.maxPerPage {
- perPage = g.maxPerPage
- }
-
- opt := &gitlab.ListProjectMergeRequestsOptions{
- ListOptions: gitlab.ListOptions{
- PerPage: perPage,
- Page: page,
- },
- }
-
- var allPRs = make([]*base.PullRequest, 0, perPage)
-
- prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, false, fmt.Errorf("error while listing merge requests: %v", err)
- }
- for _, pr := range prs {
-
- var labels = make([]*base.Label, 0, len(pr.Labels))
- for _, l := range pr.Labels {
- labels = append(labels, &base.Label{
- Name: l,
- })
- }
-
- var merged bool
- if pr.State == "merged" {
- merged = true
- pr.State = "closed"
- }
-
- var mergeTime = pr.MergedAt
- if merged && pr.MergedAt == nil {
- mergeTime = pr.UpdatedAt
- }
-
- var closeTime = pr.ClosedAt
- if merged && pr.ClosedAt == nil {
- closeTime = pr.UpdatedAt
- }
-
- var locked bool
- if pr.State == "locked" {
- locked = true
- }
-
- var milestone string
- if pr.Milestone != nil {
- milestone = pr.Milestone.Title
- }
-
- var reactions []*base.Reaction
- var awardPage = 1
- for {
- awards, _, err := g.client.AwardEmoji.ListMergeRequestAwardEmoji(g.repoID, pr.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, false, fmt.Errorf("error while listing merge requests awards: %v", err)
- }
-
- for i := range awards {
- reactions = append(reactions, g.awardToReaction(awards[i]))
- }
-
- if len(awards) < perPage {
- break
- }
-
- awardPage++
- }
-
- // Add the PR ID to the Issue Count because PR and Issues share ID space in Gitea
- newPRNumber := g.issueCount + int64(pr.IID)
-
- allPRs = append(allPRs, &base.PullRequest{
- Title: pr.Title,
- Number: newPRNumber,
- PosterName: pr.Author.Username,
- PosterID: int64(pr.Author.ID),
- Content: pr.Description,
- Milestone: milestone,
- State: pr.State,
- Created: *pr.CreatedAt,
- Closed: closeTime,
- Labels: labels,
- Merged: merged,
- MergeCommitSHA: pr.MergeCommitSHA,
- MergedTime: mergeTime,
- IsLocked: locked,
- Reactions: reactions,
- Head: base.PullRequestBranch{
- Ref: pr.SourceBranch,
- SHA: pr.SHA,
- RepoName: g.repoName,
- OwnerName: pr.Author.Username,
- CloneURL: pr.WebURL,
- },
- Base: base.PullRequestBranch{
- Ref: pr.TargetBranch,
- SHA: pr.DiffRefs.BaseSha,
- RepoName: g.repoName,
- OwnerName: pr.Author.Username,
- },
- PatchURL: pr.WebURL + ".patch",
- Context: gitlabIssueContext{
- foreignID: int64(pr.IID),
- localID: newPRNumber,
- IsMergeRequest: true,
- },
- })
- }
-
- return allPRs, len(prs) < perPage, nil
-}
-
-// GetReviews returns pull requests review
-func (g *GitlabDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
- approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(context.ForeignID()), gitlab.WithContext(g.ctx))
- if err != nil {
- if resp != nil && resp.StatusCode == 404 {
- log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error()))
- return []*base.Review{}, nil
- }
- return nil, err
- }
-
- var reviews = make([]*base.Review, 0, len(approvals.ApprovedBy))
- for _, user := range approvals.ApprovedBy {
- reviews = append(reviews, &base.Review{
- IssueIndex: context.LocalID(),
- ReviewerID: int64(user.User.ID),
- ReviewerName: user.User.Username,
- CreatedAt: *approvals.UpdatedAt,
- // All we get are approvals
- State: base.ReviewStateApproved,
- })
- }
-
- return reviews, nil
-}
-
-func (g *GitlabDownloader) awardToReaction(award *gitlab.AwardEmoji) *base.Reaction {
- return &base.Reaction{
- UserID: int64(award.User.ID),
- UserName: award.User.Username,
- Content: award.Name,
- }
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "fmt"
- "net/http"
- "os"
- "testing"
- "time"
-
- "code.gitea.io/gitea/modules/migrations/base"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestGitlabDownloadRepo(t *testing.T) {
- // Skip tests if Gitlab token is not found
- gitlabPersonalAccessToken := os.Getenv("GITLAB_READ_TOKEN")
- if gitlabPersonalAccessToken == "" {
- t.Skip("skipped test because GITLAB_READ_TOKEN was not in the environment")
- }
-
- resp, err := http.Get("https://gitlab.com/gitea/test_repo")
- if err != nil || resp.StatusCode != 200 {
- t.Skipf("Can't access test repo, skipping %s", t.Name())
- }
-
- downloader, err := NewGitlabDownloader(context.Background(), "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken)
- if err != nil {
- t.Fatal(fmt.Sprintf("NewGitlabDownloader is nil: %v", err))
- }
- repo, err := downloader.GetRepoInfo()
- assert.NoError(t, err)
- // Repo Owner is blank in Gitlab Group repos
- assertRepositoryEqual(t, &base.Repository{
- Name: "test_repo",
- Owner: "",
- Description: "Test repository for testing migration from gitlab to gitea",
- CloneURL: "https://gitlab.com/gitea/test_repo.git",
- OriginalURL: "https://gitlab.com/gitea/test_repo",
- DefaultBranch: "master",
- }, repo)
-
- topics, err := downloader.GetTopics()
- assert.NoError(t, err)
- assert.True(t, len(topics) == 2)
- assert.EqualValues(t, []string{"migration", "test"}, topics)
-
- milestones, err := downloader.GetMilestones()
- assert.NoError(t, err)
- assertMilestonesEqual(t, []*base.Milestone{
- {
- Title: "1.1.0",
- Created: time.Date(2019, 11, 28, 8, 42, 44, 575000000, time.UTC),
- Updated: timePtr(time.Date(2019, 11, 28, 8, 42, 44, 575000000, time.UTC)),
- State: "active",
- },
- {
- Title: "1.0.0",
- Created: time.Date(2019, 11, 28, 8, 42, 30, 301000000, time.UTC),
- Updated: timePtr(time.Date(2019, 11, 28, 15, 57, 52, 401000000, time.UTC)),
- Closed: timePtr(time.Date(2019, 11, 28, 15, 57, 52, 401000000, time.UTC)),
- State: "closed",
- },
- }, milestones)
-
- labels, err := downloader.GetLabels()
- assert.NoError(t, err)
- assertLabelsEqual(t, []*base.Label{
- {
- Name: "bug",
- Color: "d9534f",
- },
- {
- Name: "confirmed",
- Color: "d9534f",
- },
- {
- Name: "critical",
- Color: "d9534f",
- },
- {
- Name: "discussion",
- Color: "428bca",
- },
- {
- Name: "documentation",
- Color: "f0ad4e",
- },
- {
- Name: "duplicate",
- Color: "7f8c8d",
- },
- {
- Name: "enhancement",
- Color: "5cb85c",
- },
- {
- Name: "suggestion",
- Color: "428bca",
- },
- {
- Name: "support",
- Color: "f0ad4e",
- },
- }, labels)
-
- releases, err := downloader.GetReleases()
- assert.NoError(t, err)
- assertReleasesEqual(t, []*base.Release{
- {
- TagName: "v0.9.99",
- TargetCommitish: "0720a3ec57c1f843568298117b874319e7deee75",
- Name: "First Release",
- Body: "A test release",
- Created: time.Date(2019, 11, 28, 9, 9, 48, 840000000, time.UTC),
- PublisherID: 1241334,
- PublisherName: "lafriks",
- },
- }, releases)
-
- issues, isEnd, err := downloader.GetIssues(1, 2)
- assert.NoError(t, err)
- assert.False(t, isEnd)
-
- assertIssuesEqual(t, []*base.Issue{
- {
- Number: 1,
- Title: "Please add an animated gif icon to the merge button",
- Content: "I just want the merge button to hurt my eyes a little. :stuck_out_tongue_closed_eyes:",
- Milestone: "1.0.0",
- PosterID: 1241334,
- PosterName: "lafriks",
- State: "closed",
- Created: time.Date(2019, 11, 28, 8, 43, 35, 459000000, time.UTC),
- Updated: time.Date(2019, 11, 28, 8, 46, 23, 304000000, time.UTC),
- Labels: []*base.Label{
- {
- Name: "bug",
- },
- {
- Name: "discussion",
- },
- },
- Reactions: []*base.Reaction{
- {
- UserID: 1241334,
- UserName: "lafriks",
- Content: "thumbsup",
- },
- {
- UserID: 1241334,
- UserName: "lafriks",
- Content: "open_mouth",
- }},
- Closed: timePtr(time.Date(2019, 11, 28, 8, 46, 23, 275000000, time.UTC)),
- },
- {
- Number: 2,
- Title: "Test issue",
- Content: "This is test issue 2, do not touch!",
- Milestone: "1.1.0",
- PosterID: 1241334,
- PosterName: "lafriks",
- State: "closed",
- Created: time.Date(2019, 11, 28, 8, 44, 46, 277000000, time.UTC),
- Updated: time.Date(2019, 11, 28, 8, 45, 44, 987000000, time.UTC),
- Labels: []*base.Label{
- {
- Name: "duplicate",
- },
- },
- Reactions: []*base.Reaction{
- {
- UserID: 1241334,
- UserName: "lafriks",
- Content: "thumbsup",
- },
- {
- UserID: 1241334,
- UserName: "lafriks",
- Content: "thumbsdown",
- },
- {
- UserID: 1241334,
- UserName: "lafriks",
- Content: "laughing",
- },
- {
- UserID: 1241334,
- UserName: "lafriks",
- Content: "tada",
- },
- {
- UserID: 1241334,
- UserName: "lafriks",
- Content: "confused",
- },
- {
- UserID: 1241334,
- UserName: "lafriks",
- Content: "hearts",
- }},
- Closed: timePtr(time.Date(2019, 11, 28, 8, 45, 44, 959000000, time.UTC)),
- },
- }, issues)
-
- comments, _, err := downloader.GetComments(base.GetCommentOptions{
- Context: gitlabIssueContext{
- foreignID: 2,
- localID: 2,
- IsMergeRequest: false,
- },
- })
- assert.NoError(t, err)
- assertCommentsEqual(t, []*base.Comment{
- {
- IssueIndex: 2,
- PosterID: 1241334,
- PosterName: "lafriks",
- Created: time.Date(2019, 11, 28, 8, 44, 52, 501000000, time.UTC),
- Content: "This is a comment",
- Reactions: nil,
- },
- {
- IssueIndex: 2,
- PosterID: 1241334,
- PosterName: "lafriks",
- Created: time.Date(2019, 11, 28, 8, 45, 2, 329000000, time.UTC),
- Content: "changed milestone to %2",
- Reactions: nil,
- },
- {
- IssueIndex: 2,
- PosterID: 1241334,
- PosterName: "lafriks",
- Created: time.Date(2019, 11, 28, 8, 45, 45, 7000000, time.UTC),
- Content: "closed",
- Reactions: nil,
- },
- {
- IssueIndex: 2,
- PosterID: 1241334,
- PosterName: "lafriks",
- Created: time.Date(2019, 11, 28, 8, 45, 53, 501000000, time.UTC),
- Content: "A second comment",
- Reactions: nil,
- },
- }, comments)
-
- prs, _, err := downloader.GetPullRequests(1, 1)
- assert.NoError(t, err)
- assertPullRequestsEqual(t, []*base.PullRequest{
- {
- Number: 4,
- Title: "Test branch",
- Content: "do not merge this PR",
- Milestone: "1.0.0",
- PosterID: 1241334,
- PosterName: "lafriks",
- State: "opened",
- Created: time.Date(2019, 11, 28, 15, 56, 54, 104000000, time.UTC),
- Labels: []*base.Label{
- {
- Name: "bug",
- },
- },
- Reactions: []*base.Reaction{{
- UserID: 4575606,
- UserName: "real6543",
- Content: "thumbsup",
- }, {
- UserID: 4575606,
- UserName: "real6543",
- Content: "tada",
- }},
- PatchURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2.patch",
- Head: base.PullRequestBranch{
- Ref: "feat/test",
- CloneURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2",
- SHA: "9f733b96b98a4175276edf6a2e1231489c3bdd23",
- RepoName: "test_repo",
- OwnerName: "lafriks",
- },
- Base: base.PullRequestBranch{
- Ref: "master",
- SHA: "",
- OwnerName: "lafriks",
- RepoName: "test_repo",
- },
- Closed: nil,
- Merged: false,
- MergedTime: nil,
- MergeCommitSHA: "",
- Context: gitlabIssueContext{
- foreignID: 2,
- localID: 4,
- IsMergeRequest: true,
- },
- },
- }, prs)
-
- rvs, err := downloader.GetReviews(base.BasicIssueContext(1))
- assert.NoError(t, err)
- assertReviewsEqual(t, []*base.Review{
- {
- ReviewerID: 4102996,
- ReviewerName: "zeripath",
- CreatedAt: time.Date(2019, 11, 28, 16, 02, 8, 377000000, time.UTC),
- State: "APPROVED",
- },
- {
- ReviewerID: 527793,
- ReviewerName: "axifive",
- CreatedAt: time.Date(2019, 11, 28, 16, 02, 8, 377000000, time.UTC),
- State: "APPROVED",
- },
- }, rvs)
-
- rvs, err = downloader.GetReviews(base.BasicIssueContext(2))
- assert.NoError(t, err)
- assertReviewsEqual(t, []*base.Review{
- {
- ReviewerID: 4575606,
- ReviewerName: "real6543",
- CreatedAt: time.Date(2020, 04, 19, 19, 24, 21, 108000000, time.UTC),
- State: "APPROVED",
- },
- }, rvs)
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "crypto/tls"
- "fmt"
- "net/http"
- "net/url"
- "strings"
- "time"
-
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/proxy"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/structs"
-
- "github.com/gogs/go-gogs-client"
-)
-
-var (
- _ base.Downloader = &GogsDownloader{}
- _ base.DownloaderFactory = &GogsDownloaderFactory{}
-)
-
-func init() {
- RegisterDownloaderFactory(&GogsDownloaderFactory{})
-}
-
-// GogsDownloaderFactory defines a gogs downloader factory
-type GogsDownloaderFactory struct {
-}
-
-// New returns a Downloader related to this factory according MigrateOptions
-func (f *GogsDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
- u, err := url.Parse(opts.CloneAddr)
- if err != nil {
- return nil, err
- }
-
- baseURL := u.Scheme + "://" + u.Host
- repoNameSpace := strings.TrimSuffix(u.Path, ".git")
- repoNameSpace = strings.Trim(repoNameSpace, "/")
-
- fields := strings.Split(repoNameSpace, "/")
- if len(fields) < 2 {
- return nil, fmt.Errorf("invalid path: %s", repoNameSpace)
- }
-
- log.Trace("Create gogs downloader. BaseURL: %s RepoOwner: %s RepoName: %s", baseURL, fields[0], fields[1])
- return NewGogsDownloader(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, fields[0], fields[1]), nil
-}
-
-// GitServiceType returns the type of git service
-func (f *GogsDownloaderFactory) GitServiceType() structs.GitServiceType {
- return structs.GogsService
-}
-
-// GogsDownloader implements a Downloader interface to get repository information
-// from gogs via API
-type GogsDownloader struct {
- base.NullDownloader
- ctx context.Context
- client *gogs.Client
- baseURL string
- repoOwner string
- repoName string
- userName string
- password string
- openIssuesFinished bool
- openIssuesPages int
- transport http.RoundTripper
-}
-
-// SetContext set context
-func (g *GogsDownloader) SetContext(ctx context.Context) {
- g.ctx = ctx
-}
-
-// NewGogsDownloader creates a gogs Downloader via gogs API
-func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader {
- var downloader = GogsDownloader{
- ctx: ctx,
- baseURL: baseURL,
- userName: userName,
- password: password,
- repoOwner: repoOwner,
- repoName: repoName,
- }
-
- var client *gogs.Client
- if len(token) != 0 {
- client = gogs.NewClient(baseURL, token)
- downloader.userName = token
- } else {
- downloader.transport = &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
- Proxy: func(req *http.Request) (*url.URL, error) {
- req.SetBasicAuth(userName, password)
- return proxy.Proxy()(req)
- },
- }
-
- client = gogs.NewClient(baseURL, "")
- client.SetHTTPClient(&http.Client{
- Transport: &downloader,
- })
- }
-
- downloader.client = client
- return &downloader
-}
-
-// RoundTrip wraps the provided request within this downloader's context and passes it to our internal http.Transport.
-// This implements http.RoundTripper and makes the gogs client requests cancellable even though it is not cancellable itself
-func (g *GogsDownloader) RoundTrip(req *http.Request) (*http.Response, error) {
- return g.transport.RoundTrip(req.WithContext(g.ctx))
-}
-
-// GetRepoInfo returns a repository information
-func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) {
- gr, err := g.client.GetRepo(g.repoOwner, g.repoName)
- if err != nil {
- return nil, err
- }
-
- // convert gogs repo to stand Repo
- return &base.Repository{
- Owner: g.repoOwner,
- Name: g.repoName,
- IsPrivate: gr.Private,
- Description: gr.Description,
- CloneURL: gr.CloneURL,
- OriginalURL: gr.HTMLURL,
- DefaultBranch: gr.DefaultBranch,
- }, nil
-}
-
-// GetMilestones returns milestones
-func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) {
- var perPage = 100
- var milestones = make([]*base.Milestone, 0, perPage)
-
- ms, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName)
- if err != nil {
- return nil, err
- }
-
- for _, m := range ms {
- milestones = append(milestones, &base.Milestone{
- Title: m.Title,
- Description: m.Description,
- Deadline: m.Deadline,
- State: string(m.State),
- Closed: m.Closed,
- })
- }
-
- return milestones, nil
-}
-
-// GetLabels returns labels
-func (g *GogsDownloader) GetLabels() ([]*base.Label, error) {
- var perPage = 100
- var labels = make([]*base.Label, 0, perPage)
- ls, err := g.client.ListRepoLabels(g.repoOwner, g.repoName)
- if err != nil {
- return nil, err
- }
-
- for _, label := range ls {
- labels = append(labels, convertGogsLabel(label))
- }
-
- return labels, nil
-}
-
-// GetIssues returns issues according start and limit, perPage is not supported
-func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) {
- var state string
- if g.openIssuesFinished {
- state = string(gogs.STATE_CLOSED)
- page -= g.openIssuesPages
- } else {
- state = string(gogs.STATE_OPEN)
- g.openIssuesPages = page
- }
-
- issues, isEnd, err := g.getIssues(page, state)
- if err != nil {
- return nil, false, err
- }
-
- if isEnd {
- if g.openIssuesFinished {
- return issues, true, nil
- }
- g.openIssuesFinished = true
- }
-
- return issues, false, nil
-}
-
-func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, error) {
- var allIssues = make([]*base.Issue, 0, 10)
-
- issues, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{
- Page: page,
- State: state,
- })
- if err != nil {
- return nil, false, fmt.Errorf("error while listing repos: %v", err)
- }
-
- for _, issue := range issues {
- if issue.PullRequest != nil {
- continue
- }
- allIssues = append(allIssues, convertGogsIssue(issue))
- }
-
- return allIssues, len(issues) == 0, nil
-}
-
-// GetComments returns comments according issueNumber
-func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
- var allComments = make([]*base.Comment, 0, 100)
-
- comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID())
- if err != nil {
- return nil, false, fmt.Errorf("error while listing repos: %v", err)
- }
- for _, comment := range comments {
- if len(comment.Body) == 0 || comment.Poster == nil {
- continue
- }
- allComments = append(allComments, &base.Comment{
- IssueIndex: opts.Context.LocalID(),
- PosterID: comment.Poster.ID,
- PosterName: comment.Poster.Login,
- PosterEmail: comment.Poster.Email,
- Content: comment.Body,
- Created: comment.Created,
- Updated: comment.Updated,
- })
- }
-
- return allComments, true, nil
-}
-
-// GetTopics return repository topics
-func (g *GogsDownloader) GetTopics() ([]string, error) {
- return []string{}, nil
-}
-
-// FormatCloneURL add authentification into remote URLs
-func (g *GogsDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
- if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
- u, err := url.Parse(remoteAddr)
- if err != nil {
- return "", err
- }
- if len(opts.AuthToken) != 0 {
- u.User = url.UserPassword(opts.AuthToken, "")
- } else {
- u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
- }
- return u.String(), nil
- }
- return remoteAddr, nil
-}
-
-func convertGogsIssue(issue *gogs.Issue) *base.Issue {
- var milestone string
- if issue.Milestone != nil {
- milestone = issue.Milestone.Title
- }
- var labels = make([]*base.Label, 0, len(issue.Labels))
- for _, l := range issue.Labels {
- labels = append(labels, convertGogsLabel(l))
- }
-
- var closed *time.Time
- if issue.State == gogs.STATE_CLOSED {
- // gogs client haven't provide closed, so we use updated instead
- closed = &issue.Updated
- }
-
- return &base.Issue{
- Title: issue.Title,
- Number: issue.Index,
- PosterID: issue.Poster.ID,
- PosterName: issue.Poster.Login,
- PosterEmail: issue.Poster.Email,
- Content: issue.Body,
- Milestone: milestone,
- State: string(issue.State),
- Created: issue.Created,
- Updated: issue.Updated,
- Labels: labels,
- Closed: closed,
- Context: base.BasicIssueContext(issue.Index),
- }
-}
-
-func convertGogsLabel(label *gogs.Label) *base.Label {
- return &base.Label{
- Name: label.Name,
- Color: label.Color,
- }
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "net/http"
- "os"
- "testing"
- "time"
-
- "code.gitea.io/gitea/modules/migrations/base"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestGogsDownloadRepo(t *testing.T) {
- // Skip tests if Gogs token is not found
- gogsPersonalAccessToken := os.Getenv("GOGS_READ_TOKEN")
- if len(gogsPersonalAccessToken) == 0 {
- t.Skip("skipped test because GOGS_READ_TOKEN was not in the environment")
- }
-
- resp, err := http.Get("https://try.gogs.io/lunnytest/TESTREPO")
- if err != nil || resp.StatusCode/100 != 2 {
- // skip and don't run test
- t.Skipf("visit test repo failed, ignored")
- return
- }
-
- downloader := NewGogsDownloader(context.Background(), "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO")
- repo, err := downloader.GetRepoInfo()
- assert.NoError(t, err)
-
- assertRepositoryEqual(t, &base.Repository{
- Name: "TESTREPO",
- Owner: "lunnytest",
- Description: "",
- CloneURL: "https://try.gogs.io/lunnytest/TESTREPO.git",
- OriginalURL: "https://try.gogs.io/lunnytest/TESTREPO",
- DefaultBranch: "master",
- }, repo)
-
- milestones, err := downloader.GetMilestones()
- assert.NoError(t, err)
- assertMilestonesEqual(t, []*base.Milestone{
- {
- Title: "1.0",
- State: "open",
- },
- }, milestones)
-
- labels, err := downloader.GetLabels()
- assert.NoError(t, err)
- assertLabelsEqual(t, []*base.Label{
- {
- Name: "bug",
- Color: "ee0701",
- },
- {
- Name: "duplicate",
- Color: "cccccc",
- },
- {
- Name: "enhancement",
- Color: "84b6eb",
- },
- {
- Name: "help wanted",
- Color: "128a0c",
- },
- {
- Name: "invalid",
- Color: "e6e6e6",
- },
- {
- Name: "question",
- Color: "cc317c",
- },
- {
- Name: "wontfix",
- Color: "ffffff",
- },
- }, labels)
-
- // downloader.GetIssues()
- issues, isEnd, err := downloader.GetIssues(1, 8)
- assert.NoError(t, err)
- assert.False(t, isEnd)
- assertIssuesEqual(t, []*base.Issue{
- {
- Number: 1,
- PosterID: 5331,
- PosterName: "lunny",
- PosterEmail: "xiaolunwen@gmail.com",
- Title: "test",
- Content: "test",
- Milestone: "",
- State: "open",
- Created: time.Date(2019, 06, 11, 8, 16, 44, 0, time.UTC),
- Updated: time.Date(2019, 10, 26, 11, 07, 2, 0, time.UTC),
- Labels: []*base.Label{
- {
- Name: "bug",
- Color: "ee0701",
- },
- },
- },
- }, issues)
-
- // downloader.GetComments()
- comments, _, err := downloader.GetComments(base.GetCommentOptions{
- Context: base.BasicIssueContext(1),
- })
- assert.NoError(t, err)
- assertCommentsEqual(t, []*base.Comment{
- {
- IssueIndex: 1,
- PosterID: 5331,
- PosterName: "lunny",
- PosterEmail: "xiaolunwen@gmail.com",
- Created: time.Date(2019, 06, 11, 8, 19, 50, 0, time.UTC),
- Updated: time.Date(2019, 06, 11, 8, 19, 50, 0, time.UTC),
- Content: "1111",
- },
- {
- IssueIndex: 1,
- PosterID: 15822,
- PosterName: "clacplouf",
- PosterEmail: "test1234@dbn.re",
- Created: time.Date(2019, 10, 26, 11, 7, 2, 0, time.UTC),
- Updated: time.Date(2019, 10, 26, 11, 7, 2, 0, time.UTC),
- Content: "88888888",
- },
- }, comments)
-
- // downloader.GetPullRequests()
- _, _, err = downloader.GetPullRequests(1, 3)
- assert.Error(t, err)
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "path/filepath"
- "testing"
- "time"
-
- "code.gitea.io/gitea/models/unittest"
- "code.gitea.io/gitea/modules/migrations/base"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestMain(m *testing.M) {
- unittest.MainTest(m, filepath.Join("..", ".."))
-}
-
-func timePtr(t time.Time) *time.Time {
- return &t
-}
-
-func assertTimeEqual(t *testing.T, expected, actual time.Time) {
- assert.Equal(t, expected.UTC(), actual.UTC())
-}
-
-func assertTimePtrEqual(t *testing.T, expected, actual *time.Time) {
- if expected == nil {
- assert.Nil(t, actual)
- } else {
- assertTimeEqual(t, *expected, *actual)
- }
-}
-
-func assertCommentEqual(t *testing.T, expected, actual *base.Comment) {
- assert.Equal(t, expected.IssueIndex, actual.IssueIndex)
- assert.Equal(t, expected.PosterID, actual.PosterID)
- assert.Equal(t, expected.PosterName, actual.PosterName)
- assert.Equal(t, expected.PosterEmail, actual.PosterEmail)
- assertTimeEqual(t, expected.Created, actual.Created)
- assertTimeEqual(t, expected.Updated, actual.Updated)
- assert.Equal(t, expected.Content, actual.Content)
- assertReactionsEqual(t, expected.Reactions, actual.Reactions)
-}
-
-func assertCommentsEqual(t *testing.T, expected, actual []*base.Comment) {
- if assert.Len(t, actual, len(expected)) {
- for i := range expected {
- assertCommentEqual(t, expected[i], actual[i])
- }
- }
-}
-
-func assertLabelEqual(t *testing.T, expected, actual *base.Label) {
- assert.Equal(t, expected.Name, actual.Name)
- assert.Equal(t, expected.Color, actual.Color)
- assert.Equal(t, expected.Description, actual.Description)
-}
-
-func assertLabelsEqual(t *testing.T, expected, actual []*base.Label) {
- if assert.Len(t, actual, len(expected)) {
- for i := range expected {
- assertLabelEqual(t, expected[i], actual[i])
- }
- }
-}
-
-func assertMilestoneEqual(t *testing.T, expected, actual *base.Milestone) {
- assert.Equal(t, expected.Title, actual.Title)
- assert.Equal(t, expected.Description, actual.Description)
- assertTimePtrEqual(t, expected.Deadline, actual.Deadline)
- assertTimeEqual(t, expected.Created, actual.Created)
- assertTimePtrEqual(t, expected.Updated, actual.Updated)
- assertTimePtrEqual(t, expected.Closed, actual.Closed)
- assert.Equal(t, expected.State, actual.State)
-}
-
-func assertMilestonesEqual(t *testing.T, expected, actual []*base.Milestone) {
- if assert.Len(t, actual, len(expected)) {
- for i := range expected {
- assertMilestoneEqual(t, expected[i], actual[i])
- }
- }
-}
-
-func assertIssueEqual(t *testing.T, expected, actual *base.Issue) {
- assert.Equal(t, expected.Number, actual.Number)
- assert.Equal(t, expected.PosterID, actual.PosterID)
- assert.Equal(t, expected.PosterName, actual.PosterName)
- assert.Equal(t, expected.PosterEmail, actual.PosterEmail)
- assert.Equal(t, expected.Title, actual.Title)
- assert.Equal(t, expected.Content, actual.Content)
- assert.Equal(t, expected.Ref, actual.Ref)
- assert.Equal(t, expected.Milestone, actual.Milestone)
- assert.Equal(t, expected.State, actual.State)
- assert.Equal(t, expected.IsLocked, actual.IsLocked)
- assertTimeEqual(t, expected.Created, actual.Created)
- assertTimeEqual(t, expected.Updated, actual.Updated)
- assertTimePtrEqual(t, expected.Closed, actual.Closed)
- assertLabelsEqual(t, expected.Labels, actual.Labels)
- assertReactionsEqual(t, expected.Reactions, actual.Reactions)
- assert.ElementsMatch(t, expected.Assignees, actual.Assignees)
-}
-
-func assertIssuesEqual(t *testing.T, expected, actual []*base.Issue) {
- if assert.Len(t, actual, len(expected)) {
- for i := range expected {
- assertIssueEqual(t, expected[i], actual[i])
- }
- }
-}
-
-func assertPullRequestEqual(t *testing.T, expected, actual *base.PullRequest) {
- assert.Equal(t, expected.Number, actual.Number)
- assert.Equal(t, expected.Title, actual.Title)
- assert.Equal(t, expected.PosterID, actual.PosterID)
- assert.Equal(t, expected.PosterName, actual.PosterName)
- assert.Equal(t, expected.PosterEmail, actual.PosterEmail)
- assert.Equal(t, expected.Content, actual.Content)
- assert.Equal(t, expected.Milestone, actual.Milestone)
- assert.Equal(t, expected.State, actual.State)
- assertTimeEqual(t, expected.Created, actual.Created)
- assertTimeEqual(t, expected.Updated, actual.Updated)
- assertTimePtrEqual(t, expected.Closed, actual.Closed)
- assertLabelsEqual(t, expected.Labels, actual.Labels)
- assert.Equal(t, expected.PatchURL, actual.PatchURL)
- assert.Equal(t, expected.Merged, actual.Merged)
- assertTimePtrEqual(t, expected.MergedTime, actual.MergedTime)
- assert.Equal(t, expected.MergeCommitSHA, actual.MergeCommitSHA)
- assertPullRequestBranchEqual(t, expected.Head, actual.Head)
- assertPullRequestBranchEqual(t, expected.Base, actual.Base)
- assert.ElementsMatch(t, expected.Assignees, actual.Assignees)
- assert.Equal(t, expected.IsLocked, actual.IsLocked)
- assertReactionsEqual(t, expected.Reactions, actual.Reactions)
-}
-
-func assertPullRequestsEqual(t *testing.T, expected, actual []*base.PullRequest) {
- if assert.Len(t, actual, len(expected)) {
- for i := range expected {
- assertPullRequestEqual(t, expected[i], actual[i])
- }
- }
-}
-
-func assertPullRequestBranchEqual(t *testing.T, expected, actual base.PullRequestBranch) {
- assert.Equal(t, expected.CloneURL, actual.CloneURL)
- assert.Equal(t, expected.Ref, actual.Ref)
- assert.Equal(t, expected.SHA, actual.SHA)
- assert.Equal(t, expected.RepoName, actual.RepoName)
- assert.Equal(t, expected.OwnerName, actual.OwnerName)
-}
-
-func assertReactionEqual(t *testing.T, expected, actual *base.Reaction) {
- assert.Equal(t, expected.UserID, actual.UserID)
- assert.Equal(t, expected.UserName, actual.UserName)
- assert.Equal(t, expected.Content, actual.Content)
-}
-
-func assertReactionsEqual(t *testing.T, expected, actual []*base.Reaction) {
- if assert.Len(t, actual, len(expected)) {
- for i := range expected {
- assertReactionEqual(t, expected[i], actual[i])
- }
- }
-}
-
-func assertReleaseAssetEqual(t *testing.T, expected, actual *base.ReleaseAsset) {
- assert.Equal(t, expected.ID, actual.ID)
- assert.Equal(t, expected.Name, actual.Name)
- assert.Equal(t, expected.ContentType, actual.ContentType)
- assert.Equal(t, expected.Size, actual.Size)
- assert.Equal(t, expected.DownloadCount, actual.DownloadCount)
- assertTimeEqual(t, expected.Created, actual.Created)
- assertTimeEqual(t, expected.Updated, actual.Updated)
- assert.Equal(t, expected.DownloadURL, actual.DownloadURL)
-}
-
-func assertReleaseAssetsEqual(t *testing.T, expected, actual []*base.ReleaseAsset) {
- if assert.Len(t, actual, len(expected)) {
- for i := range expected {
- assertReleaseAssetEqual(t, expected[i], actual[i])
- }
- }
-}
-
-func assertReleaseEqual(t *testing.T, expected, actual *base.Release) {
- assert.Equal(t, expected.TagName, actual.TagName)
- assert.Equal(t, expected.TargetCommitish, actual.TargetCommitish)
- assert.Equal(t, expected.Name, actual.Name)
- assert.Equal(t, expected.Body, actual.Body)
- assert.Equal(t, expected.Draft, actual.Draft)
- assert.Equal(t, expected.Prerelease, actual.Prerelease)
- assert.Equal(t, expected.PublisherID, actual.PublisherID)
- assert.Equal(t, expected.PublisherName, actual.PublisherName)
- assert.Equal(t, expected.PublisherEmail, actual.PublisherEmail)
- assertReleaseAssetsEqual(t, expected.Assets, actual.Assets)
- assertTimeEqual(t, expected.Created, actual.Created)
- assertTimeEqual(t, expected.Published, actual.Published)
-}
-
-func assertReleasesEqual(t *testing.T, expected, actual []*base.Release) {
- if assert.Len(t, actual, len(expected)) {
- for i := range expected {
- assertReleaseEqual(t, expected[i], actual[i])
- }
- }
-}
-
-func assertRepositoryEqual(t *testing.T, expected, actual *base.Repository) {
- assert.Equal(t, expected.Name, actual.Name)
- assert.Equal(t, expected.Owner, actual.Owner)
- assert.Equal(t, expected.IsPrivate, actual.IsPrivate)
- assert.Equal(t, expected.IsMirror, actual.IsMirror)
- assert.Equal(t, expected.Description, actual.Description)
- assert.Equal(t, expected.CloneURL, actual.CloneURL)
- assert.Equal(t, expected.OriginalURL, actual.OriginalURL)
- assert.Equal(t, expected.DefaultBranch, actual.DefaultBranch)
-}
-
-func assertReviewEqual(t *testing.T, expected, actual *base.Review) {
- assert.Equal(t, expected.ID, actual.ID)
- assert.Equal(t, expected.IssueIndex, actual.IssueIndex)
- assert.Equal(t, expected.ReviewerID, actual.ReviewerID)
- assert.Equal(t, expected.ReviewerName, actual.ReviewerName)
- assert.Equal(t, expected.Official, actual.Official)
- assert.Equal(t, expected.CommitID, actual.CommitID)
- assert.Equal(t, expected.Content, actual.Content)
- assertTimeEqual(t, expected.CreatedAt, actual.CreatedAt)
- assert.Equal(t, expected.State, actual.State)
- assertReviewCommentsEqual(t, expected.Comments, actual.Comments)
-}
-
-func assertReviewsEqual(t *testing.T, expected, actual []*base.Review) {
- if assert.Len(t, actual, len(expected)) {
- for i := range expected {
- assertReviewEqual(t, expected[i], actual[i])
- }
- }
-}
-
-func assertReviewCommentEqual(t *testing.T, expected, actual *base.ReviewComment) {
- assert.Equal(t, expected.ID, actual.ID)
- assert.Equal(t, expected.InReplyTo, actual.InReplyTo)
- assert.Equal(t, expected.Content, actual.Content)
- assert.Equal(t, expected.TreePath, actual.TreePath)
- assert.Equal(t, expected.DiffHunk, actual.DiffHunk)
- assert.Equal(t, expected.Position, actual.Position)
- assert.Equal(t, expected.Line, actual.Line)
- assert.Equal(t, expected.CommitID, actual.CommitID)
- assert.Equal(t, expected.PosterID, actual.PosterID)
- assertReactionsEqual(t, expected.Reactions, actual.Reactions)
- assertTimeEqual(t, expected.CreatedAt, actual.CreatedAt)
- assertTimeEqual(t, expected.UpdatedAt, actual.UpdatedAt)
-}
-
-func assertReviewCommentsEqual(t *testing.T, expected, actual []*base.ReviewComment) {
- if assert.Len(t, actual, len(expected)) {
- for i := range expected {
- assertReviewCommentEqual(t, expected[i], actual[i])
- }
- }
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Copyright 2018 Jonas Franz. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "fmt"
- "net"
- "net/url"
- "path/filepath"
- "strings"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/matchlist"
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/util"
-)
-
-// MigrateOptions is equal to base.MigrateOptions
-type MigrateOptions = base.MigrateOptions
-
-var (
- factories []base.DownloaderFactory
-
- allowList *matchlist.Matchlist
- blockList *matchlist.Matchlist
-)
-
-// RegisterDownloaderFactory registers a downloader factory
-func RegisterDownloaderFactory(factory base.DownloaderFactory) {
- factories = append(factories, factory)
-}
-
-// IsMigrateURLAllowed checks if an URL is allowed to be migrated from
-func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
- // Remote address can be HTTP/HTTPS/Git URL or local path.
- u, err := url.Parse(remoteURL)
- if err != nil {
- return &models.ErrInvalidCloneAddr{IsURLError: true}
- }
-
- if u.Scheme == "file" || u.Scheme == "" {
- if !doer.CanImportLocal() {
- return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsPermissionDenied: true, LocalPath: true}
- }
- isAbs := filepath.IsAbs(u.Host + u.Path)
- if !isAbs {
- return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
- }
- isDir, err := util.IsDir(u.Host + u.Path)
- if err != nil {
- log.Error("Unable to check if %s is a directory: %v", u.Host+u.Path, err)
- return err
- }
- if !isDir {
- return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
- }
-
- return nil
- }
-
- if u.Scheme == "git" && u.Port() != "" && (strings.Contains(remoteURL, "%0d") || strings.Contains(remoteURL, "%0a")) {
- return &models.ErrInvalidCloneAddr{Host: u.Host, IsURLError: true}
- }
-
- if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" {
- return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true}
- }
-
- host := strings.ToLower(u.Host)
- if len(setting.Migrations.AllowedDomains) > 0 {
- if !allowList.Match(host) {
- return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
- }
- } else {
- if blockList.Match(host) {
- return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
- }
- }
-
- if !setting.Migrations.AllowLocalNetworks {
- addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0])
- if err != nil {
- return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true}
- }
- for _, addr := range addrList {
- if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() {
- return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true}
- }
- }
- }
-
- return nil
-}
-
-// MigrateRepository migrate repository according MigrateOptions
-func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*models.Repository, error) {
- err := IsMigrateURLAllowed(opts.CloneAddr, doer)
- if err != nil {
- return nil, err
- }
- if opts.LFS && len(opts.LFSEndpoint) > 0 {
- err := IsMigrateURLAllowed(opts.LFSEndpoint, doer)
- if err != nil {
- return nil, err
- }
- }
- downloader, err := newDownloader(ctx, ownerName, opts)
- if err != nil {
- return nil, err
- }
-
- var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
- uploader.gitServiceType = opts.GitServiceType
-
- if err := migrateRepository(downloader, uploader, opts, messenger); err != nil {
- if err1 := uploader.Rollback(); err1 != nil {
- log.Error("rollback failed: %v", err1)
- }
- if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
- log.Error("create respotiry notice failed: ", err2)
- }
- return nil, err
- }
- return uploader.repo, nil
-}
-
-func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) {
- var (
- downloader base.Downloader
- err error
- )
-
- for _, factory := range factories {
- if factory.GitServiceType() == opts.GitServiceType {
- downloader, err = factory.New(ctx, opts)
- if err != nil {
- return nil, err
- }
- break
- }
- }
-
- if downloader == nil {
- opts.Wiki = true
- opts.Milestones = false
- opts.Labels = false
- opts.Releases = false
- opts.Comments = false
- opts.Issues = false
- opts.PullRequests = false
- downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
- log.Trace("Will migrate from git: %s", opts.OriginalURL)
- }
-
- if setting.Migrations.MaxAttempts > 1 {
- downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
- }
- return downloader, nil
-}
-
-// migrateRepository will download information and then upload it to Uploader, this is a simple
-// process for small repository. For a big repository, save all the data to disk
-// before upload is better
-func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
- if messenger == nil {
- messenger = base.NilMessenger
- }
-
- repo, err := downloader.GetRepoInfo()
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Info("migrating repo infos is not supported, ignored")
- }
- repo.IsPrivate = opts.Private
- repo.IsMirror = opts.Mirror
- if opts.Description != "" {
- repo.Description = opts.Description
- }
- if repo.CloneURL, err = downloader.FormatCloneURL(opts, repo.CloneURL); err != nil {
- return err
- }
-
- log.Trace("migrating git data from %s", repo.CloneURL)
- messenger("repo.migrate.migrating_git")
- if err = uploader.CreateRepo(repo, opts); err != nil {
- return err
- }
- defer uploader.Close()
-
- log.Trace("migrating topics")
- messenger("repo.migrate.migrating_topics")
- topics, err := downloader.GetTopics()
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating topics is not supported, ignored")
- }
- if len(topics) != 0 {
- if err = uploader.CreateTopics(topics...); err != nil {
- return err
- }
- }
-
- if opts.Milestones {
- log.Trace("migrating milestones")
- messenger("repo.migrate.migrating_milestones")
- milestones, err := downloader.GetMilestones()
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating milestones is not supported, ignored")
- }
-
- msBatchSize := uploader.MaxBatchInsertSize("milestone")
- for len(milestones) > 0 {
- if len(milestones) < msBatchSize {
- msBatchSize = len(milestones)
- }
-
- if err := uploader.CreateMilestones(milestones...); err != nil {
- return err
- }
- milestones = milestones[msBatchSize:]
- }
- }
-
- if opts.Labels {
- log.Trace("migrating labels")
- messenger("repo.migrate.migrating_labels")
- labels, err := downloader.GetLabels()
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating labels is not supported, ignored")
- }
-
- lbBatchSize := uploader.MaxBatchInsertSize("label")
- for len(labels) > 0 {
- if len(labels) < lbBatchSize {
- lbBatchSize = len(labels)
- }
-
- if err := uploader.CreateLabels(labels...); err != nil {
- return err
- }
- labels = labels[lbBatchSize:]
- }
- }
-
- if opts.Releases {
- log.Trace("migrating releases")
- messenger("repo.migrate.migrating_releases")
- releases, err := downloader.GetReleases()
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating releases is not supported, ignored")
- }
-
- relBatchSize := uploader.MaxBatchInsertSize("release")
- for len(releases) > 0 {
- if len(releases) < relBatchSize {
- relBatchSize = len(releases)
- }
-
- if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
- return err
- }
- releases = releases[relBatchSize:]
- }
-
- // Once all releases (if any) are inserted, sync any remaining non-release tags
- if err = uploader.SyncTags(); err != nil {
- return err
- }
- }
-
- var (
- commentBatchSize = uploader.MaxBatchInsertSize("comment")
- reviewBatchSize = uploader.MaxBatchInsertSize("review")
- )
-
- supportAllComments := downloader.SupportGetRepoComments()
-
- if opts.Issues {
- log.Trace("migrating issues and comments")
- messenger("repo.migrate.migrating_issues")
- var issueBatchSize = uploader.MaxBatchInsertSize("issue")
-
- for i := 1; ; i++ {
- issues, isEnd, err := downloader.GetIssues(i, issueBatchSize)
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating issues is not supported, ignored")
- break
- }
-
- if err := uploader.CreateIssues(issues...); err != nil {
- return err
- }
-
- if opts.Comments && !supportAllComments {
- var allComments = make([]*base.Comment, 0, commentBatchSize)
- for _, issue := range issues {
- log.Trace("migrating issue %d's comments", issue.Number)
- comments, _, err := downloader.GetComments(base.GetCommentOptions{
- Context: issue.Context,
- })
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating comments is not supported, ignored")
- }
-
- allComments = append(allComments, comments...)
-
- if len(allComments) >= commentBatchSize {
- if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
- return err
- }
-
- allComments = allComments[commentBatchSize:]
- }
- }
-
- if len(allComments) > 0 {
- if err = uploader.CreateComments(allComments...); err != nil {
- return err
- }
- }
- }
-
- if isEnd {
- break
- }
- }
- }
-
- if opts.PullRequests {
- log.Trace("migrating pull requests and comments")
- messenger("repo.migrate.migrating_pulls")
- var prBatchSize = uploader.MaxBatchInsertSize("pullrequest")
- for i := 1; ; i++ {
- prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating pull requests is not supported, ignored")
- break
- }
-
- if err := uploader.CreatePullRequests(prs...); err != nil {
- return err
- }
-
- if opts.Comments {
- if !supportAllComments {
- // plain comments
- var allComments = make([]*base.Comment, 0, commentBatchSize)
- for _, pr := range prs {
- log.Trace("migrating pull request %d's comments", pr.Number)
- comments, _, err := downloader.GetComments(base.GetCommentOptions{
- Context: pr.Context,
- })
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating comments is not supported, ignored")
- }
-
- allComments = append(allComments, comments...)
-
- if len(allComments) >= commentBatchSize {
- if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
- return err
- }
- allComments = allComments[commentBatchSize:]
- }
- }
- if len(allComments) > 0 {
- if err = uploader.CreateComments(allComments...); err != nil {
- return err
- }
- }
- }
-
- // migrate reviews
- var allReviews = make([]*base.Review, 0, reviewBatchSize)
- for _, pr := range prs {
- reviews, err := downloader.GetReviews(pr.Context)
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating reviews is not supported, ignored")
- break
- }
-
- allReviews = append(allReviews, reviews...)
-
- if len(allReviews) >= reviewBatchSize {
- if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
- return err
- }
- allReviews = allReviews[reviewBatchSize:]
- }
- }
- if len(allReviews) > 0 {
- if err = uploader.CreateReviews(allReviews...); err != nil {
- return err
- }
- }
- }
-
- if isEnd {
- break
- }
- }
- }
-
- if opts.Comments && supportAllComments {
- log.Trace("migrating comments")
- for i := 1; ; i++ {
- comments, isEnd, err := downloader.GetComments(base.GetCommentOptions{
- Page: i,
- PageSize: commentBatchSize,
- })
- if err != nil {
- return err
- }
-
- if err := uploader.CreateComments(comments...); err != nil {
- return err
- }
-
- if isEnd {
- break
- }
- }
- }
-
- return uploader.Finish()
-}
-
-// Init migrations service
-func Init() error {
- var err error
- allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...)
- if err != nil {
- return fmt.Errorf("init migration allowList domains failed: %v", err)
- }
-
- blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...)
- if err != nil {
- return fmt.Errorf("init migration blockList domains failed: %v", err)
- }
-
- return nil
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "path/filepath"
- "testing"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/models/unittest"
- "code.gitea.io/gitea/modules/setting"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestMigrateWhiteBlocklist(t *testing.T) {
- assert.NoError(t, unittest.PrepareTestDatabase())
-
- adminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user1"}).(*models.User)
- nonAdminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user2"}).(*models.User)
-
- setting.Migrations.AllowedDomains = []string{"github.com"}
- assert.NoError(t, Init())
-
- err := IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
- assert.Error(t, err)
-
- err = IsMigrateURLAllowed("https://github.com/go-gitea/gitea.git", nonAdminUser)
- assert.NoError(t, err)
-
- err = IsMigrateURLAllowed("https://gITHUb.com/go-gitea/gitea.git", nonAdminUser)
- assert.NoError(t, err)
-
- setting.Migrations.AllowedDomains = []string{}
- setting.Migrations.BlockedDomains = []string{"github.com"}
- assert.NoError(t, Init())
-
- err = IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
- assert.NoError(t, err)
-
- err = IsMigrateURLAllowed("https://github.com/go-gitea/gitea.git", nonAdminUser)
- assert.Error(t, err)
-
- err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
- assert.Error(t, err)
-
- setting.Migrations.AllowLocalNetworks = true
- err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
- assert.NoError(t, err)
-
- old := setting.ImportLocalPaths
- setting.ImportLocalPaths = false
-
- err = IsMigrateURLAllowed("/home/foo/bar/goo", adminUser)
- assert.Error(t, err)
-
- setting.ImportLocalPaths = true
- abs, err := filepath.Abs(".")
- assert.NoError(t, err)
-
- err = IsMigrateURLAllowed(abs, adminUser)
- assert.NoError(t, err)
-
- err = IsMigrateURLAllowed(abs, nonAdminUser)
- assert.Error(t, err)
-
- nonAdminUser.AllowImportLocal = true
- err = IsMigrateURLAllowed(abs, nonAdminUser)
- assert.NoError(t, err)
-
- setting.ImportLocalPaths = old
-}
+++ /dev/null
-// Copyright 2021 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "fmt"
- "net/http"
- "net/url"
- "strconv"
- "strings"
- "time"
-
- "code.gitea.io/gitea/modules/json"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/structs"
-)
-
-var (
- _ base.Downloader = &OneDevDownloader{}
- _ base.DownloaderFactory = &OneDevDownloaderFactory{}
-)
-
-func init() {
- RegisterDownloaderFactory(&OneDevDownloaderFactory{})
-}
-
-// OneDevDownloaderFactory defines a downloader factory
-type OneDevDownloaderFactory struct {
-}
-
-// New returns a downloader related to this factory according MigrateOptions
-func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
- u, err := url.Parse(opts.CloneAddr)
- if err != nil {
- return nil, err
- }
-
- repoName := ""
-
- fields := strings.Split(strings.Trim(u.Path, "/"), "/")
- if len(fields) == 2 && fields[0] == "projects" {
- repoName = fields[1]
- } else if len(fields) == 1 {
- repoName = fields[0]
- } else {
- return nil, fmt.Errorf("invalid path: %s", u.Path)
- }
-
- u.Path = ""
- u.Fragment = ""
-
- log.Trace("Create onedev downloader. BaseURL: %v RepoName: %s", u, repoName)
-
- return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoName), nil
-}
-
-// GitServiceType returns the type of git service
-func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType {
- return structs.OneDevService
-}
-
-type onedevUser struct {
- ID int64 `json:"id"`
- Name string `json:"name"`
- Email string `json:"email"`
-}
-
-// OneDevDownloader implements a Downloader interface to get repository informations
-// from OneDev
-type OneDevDownloader struct {
- base.NullDownloader
- ctx context.Context
- client *http.Client
- baseURL *url.URL
- repoName string
- repoID int64
- maxIssueIndex int64
- userMap map[int64]*onedevUser
- milestoneMap map[int64]string
-}
-
-// SetContext set context
-func (d *OneDevDownloader) SetContext(ctx context.Context) {
- d.ctx = ctx
-}
-
-// NewOneDevDownloader creates a new downloader
-func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader {
- var downloader = &OneDevDownloader{
- ctx: ctx,
- baseURL: baseURL,
- repoName: repoName,
- client: &http.Client{
- Transport: &http.Transport{
- Proxy: func(req *http.Request) (*url.URL, error) {
- if len(username) > 0 && len(password) > 0 {
- req.SetBasicAuth(username, password)
- }
- return nil, nil
- },
- },
- },
- userMap: make(map[int64]*onedevUser),
- milestoneMap: make(map[int64]string),
- }
-
- return downloader
-}
-
-func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error {
- u, err := d.baseURL.Parse(endpoint)
- if err != nil {
- return err
- }
-
- if parameter != nil {
- query := u.Query()
- for k, v := range parameter {
- query.Set(k, v)
- }
- u.RawQuery = query.Encode()
- }
-
- req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil)
- if err != nil {
- return err
- }
-
- resp, err := d.client.Do(req)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
-
- decoder := json.NewDecoder(resp.Body)
- return decoder.Decode(&result)
-}
-
-// GetRepoInfo returns repository information
-func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) {
- info := make([]struct {
- ID int64 `json:"id"`
- Name string `json:"name"`
- Description string `json:"description"`
- }, 0, 1)
-
- err := d.callAPI(
- "/api/projects",
- map[string]string{
- "query": `"Name" is "` + d.repoName + `"`,
- "offset": "0",
- "count": "1",
- },
- &info,
- )
- if err != nil {
- return nil, err
- }
- if len(info) != 1 {
- return nil, fmt.Errorf("Project %s not found", d.repoName)
- }
-
- d.repoID = info[0].ID
-
- cloneURL, err := d.baseURL.Parse(info[0].Name)
- if err != nil {
- return nil, err
- }
- originalURL, err := d.baseURL.Parse("/projects/" + info[0].Name)
- if err != nil {
- return nil, err
- }
-
- return &base.Repository{
- Name: info[0].Name,
- Description: info[0].Description,
- CloneURL: cloneURL.String(),
- OriginalURL: originalURL.String(),
- }, nil
-}
-
-// GetMilestones returns milestones
-func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) {
- rawMilestones := make([]struct {
- ID int64 `json:"id"`
- Name string `json:"name"`
- Description string `json:"description"`
- DueDate *time.Time `json:"dueDate"`
- Closed bool `json:"closed"`
- }, 0, 100)
-
- endpoint := fmt.Sprintf("/api/projects/%d/milestones", d.repoID)
-
- var milestones = make([]*base.Milestone, 0, 100)
- offset := 0
- for {
- err := d.callAPI(
- endpoint,
- map[string]string{
- "offset": strconv.Itoa(offset),
- "count": "100",
- },
- &rawMilestones,
- )
- if err != nil {
- return nil, err
- }
- if len(rawMilestones) == 0 {
- break
- }
- offset += 100
-
- for _, milestone := range rawMilestones {
- d.milestoneMap[milestone.ID] = milestone.Name
- closed := milestone.DueDate
- if !milestone.Closed {
- closed = nil
- }
-
- milestones = append(milestones, &base.Milestone{
- Title: milestone.Name,
- Description: milestone.Description,
- Deadline: milestone.DueDate,
- Closed: closed,
- })
- }
- }
- return milestones, nil
-}
-
-// GetLabels returns labels
-func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) {
- return []*base.Label{
- {
- Name: "Bug",
- Color: "f64e60",
- },
- {
- Name: "Build Failure",
- Color: "f64e60",
- },
- {
- Name: "Discussion",
- Color: "8950fc",
- },
- {
- Name: "Improvement",
- Color: "1bc5bd",
- },
- {
- Name: "New Feature",
- Color: "1bc5bd",
- },
- {
- Name: "Support Request",
- Color: "8950fc",
- },
- }, nil
-}
-
-type onedevIssueContext struct {
- foreignID int64
- localID int64
- IsPullRequest bool
-}
-
-func (c onedevIssueContext) LocalID() int64 {
- return c.localID
-}
-
-func (c onedevIssueContext) ForeignID() int64 {
- return c.foreignID
-}
-
-// GetIssues returns issues
-func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
- rawIssues := make([]struct {
- ID int64 `json:"id"`
- Number int64 `json:"number"`
- State string `json:"state"`
- Title string `json:"title"`
- Description string `json:"description"`
- MilestoneID int64 `json:"milestoneId"`
- SubmitterID int64 `json:"submitterId"`
- SubmitDate time.Time `json:"submitDate"`
- }, 0, perPage)
-
- err := d.callAPI(
- "/api/issues",
- map[string]string{
- "query": `"Project" is "` + d.repoName + `"`,
- "offset": strconv.Itoa((page - 1) * perPage),
- "count": strconv.Itoa(perPage),
- },
- &rawIssues,
- )
- if err != nil {
- return nil, false, err
- }
-
- issues := make([]*base.Issue, 0, len(rawIssues))
- for _, issue := range rawIssues {
- fields := make([]struct {
- Name string `json:"name"`
- Value string `json:"value"`
- }, 0, 10)
- err := d.callAPI(
- fmt.Sprintf("/api/issues/%d/fields", issue.ID),
- nil,
- &fields,
- )
- if err != nil {
- return nil, false, err
- }
-
- var label *base.Label
- for _, field := range fields {
- if field.Name == "Type" {
- label = &base.Label{Name: field.Value}
- break
- }
- }
-
- state := strings.ToLower(issue.State)
- if state == "released" {
- state = "closed"
- }
- poster := d.tryGetUser(issue.SubmitterID)
- issues = append(issues, &base.Issue{
- Title: issue.Title,
- Number: issue.Number,
- PosterName: poster.Name,
- PosterEmail: poster.Email,
- Content: issue.Description,
- Milestone: d.milestoneMap[issue.MilestoneID],
- State: state,
- Created: issue.SubmitDate,
- Updated: issue.SubmitDate,
- Labels: []*base.Label{label},
- Context: onedevIssueContext{
- foreignID: issue.ID,
- localID: issue.Number,
- IsPullRequest: false,
- },
- })
-
- if d.maxIssueIndex < issue.Number {
- d.maxIssueIndex = issue.Number
- }
- }
-
- return issues, len(issues) == 0, nil
-}
-
-// GetComments returns comments
-func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
- context, ok := opts.Context.(onedevIssueContext)
- if !ok {
- return nil, false, fmt.Errorf("unexpected comment context: %+v", opts.Context)
- }
-
- rawComments := make([]struct {
- Date time.Time `json:"date"`
- UserID int64 `json:"userId"`
- Content string `json:"content"`
- }, 0, 100)
-
- var endpoint string
- if context.IsPullRequest {
- endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", context.ForeignID())
- } else {
- endpoint = fmt.Sprintf("/api/issues/%d/comments", context.ForeignID())
- }
-
- err := d.callAPI(
- endpoint,
- nil,
- &rawComments,
- )
- if err != nil {
- return nil, false, err
- }
-
- rawChanges := make([]struct {
- Date time.Time `json:"date"`
- UserID int64 `json:"userId"`
- Data map[string]interface{} `json:"data"`
- }, 0, 100)
-
- if context.IsPullRequest {
- endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", context.ForeignID())
- } else {
- endpoint = fmt.Sprintf("/api/issues/%d/changes", context.ForeignID())
- }
-
- err = d.callAPI(
- endpoint,
- nil,
- &rawChanges,
- )
- if err != nil {
- return nil, false, err
- }
-
- comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges))
- for _, comment := range rawComments {
- if len(comment.Content) == 0 {
- continue
- }
- poster := d.tryGetUser(comment.UserID)
- comments = append(comments, &base.Comment{
- IssueIndex: context.LocalID(),
- PosterID: poster.ID,
- PosterName: poster.Name,
- PosterEmail: poster.Email,
- Content: comment.Content,
- Created: comment.Date,
- Updated: comment.Date,
- })
- }
- for _, change := range rawChanges {
- contentV, ok := change.Data["content"]
- if !ok {
- contentV, ok = change.Data["comment"]
- if !ok {
- continue
- }
- }
- content, ok := contentV.(string)
- if !ok || len(content) == 0 {
- continue
- }
-
- poster := d.tryGetUser(change.UserID)
- comments = append(comments, &base.Comment{
- IssueIndex: context.LocalID(),
- PosterID: poster.ID,
- PosterName: poster.Name,
- PosterEmail: poster.Email,
- Content: content,
- Created: change.Date,
- Updated: change.Date,
- })
- }
-
- return comments, true, nil
-}
-
-// GetPullRequests returns pull requests
-func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
- rawPullRequests := make([]struct {
- ID int64 `json:"id"`
- Number int64 `json:"number"`
- Title string `json:"title"`
- SubmitterID int64 `json:"submitterId"`
- SubmitDate time.Time `json:"submitDate"`
- Description string `json:"description"`
- TargetBranch string `json:"targetBranch"`
- SourceBranch string `json:"sourceBranch"`
- BaseCommitHash string `json:"baseCommitHash"`
- CloseInfo *struct {
- Date *time.Time `json:"date"`
- Status string `json:"status"`
- }
- }, 0, perPage)
-
- err := d.callAPI(
- "/api/pull-requests",
- map[string]string{
- "query": `"Target Project" is "` + d.repoName + `"`,
- "offset": strconv.Itoa((page - 1) * perPage),
- "count": strconv.Itoa(perPage),
- },
- &rawPullRequests,
- )
- if err != nil {
- return nil, false, err
- }
-
- pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests))
- for _, pr := range rawPullRequests {
- var mergePreview struct {
- TargetHeadCommitHash string `json:"targetHeadCommitHash"`
- HeadCommitHash string `json:"headCommitHash"`
- MergeStrategy string `json:"mergeStrategy"`
- MergeCommitHash string `json:"mergeCommitHash"`
- }
- err := d.callAPI(
- fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID),
- nil,
- &mergePreview,
- )
- if err != nil {
- return nil, false, err
- }
-
- state := "open"
- merged := false
- var closeTime *time.Time
- var mergedTime *time.Time
- if pr.CloseInfo != nil {
- state = "closed"
- closeTime = pr.CloseInfo.Date
- if pr.CloseInfo.Status == "MERGED" { // "DISCARDED"
- merged = true
- mergedTime = pr.CloseInfo.Date
- }
- }
- poster := d.tryGetUser(pr.SubmitterID)
-
- number := pr.Number + d.maxIssueIndex
- pullRequests = append(pullRequests, &base.PullRequest{
- Title: pr.Title,
- Number: number,
- PosterName: poster.Name,
- PosterID: poster.ID,
- Content: pr.Description,
- State: state,
- Created: pr.SubmitDate,
- Updated: pr.SubmitDate,
- Closed: closeTime,
- Merged: merged,
- MergedTime: mergedTime,
- Head: base.PullRequestBranch{
- Ref: pr.SourceBranch,
- SHA: mergePreview.HeadCommitHash,
- RepoName: d.repoName,
- },
- Base: base.PullRequestBranch{
- Ref: pr.TargetBranch,
- SHA: mergePreview.TargetHeadCommitHash,
- RepoName: d.repoName,
- },
- Context: onedevIssueContext{
- foreignID: pr.ID,
- localID: number,
- IsPullRequest: true,
- },
- })
- }
-
- return pullRequests, len(pullRequests) == 0, nil
-}
-
-// GetReviews returns pull requests reviews
-func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
- rawReviews := make([]struct {
- ID int64 `json:"id"`
- UserID int64 `json:"userId"`
- Result *struct {
- Commit string `json:"commit"`
- Approved bool `json:"approved"`
- Comment string `json:"comment"`
- }
- }, 0, 100)
-
- err := d.callAPI(
- fmt.Sprintf("/api/pull-requests/%d/reviews", context.ForeignID()),
- nil,
- &rawReviews,
- )
- if err != nil {
- return nil, err
- }
-
- var reviews = make([]*base.Review, 0, len(rawReviews))
- for _, review := range rawReviews {
- state := base.ReviewStatePending
- content := ""
- if review.Result != nil {
- if len(review.Result.Comment) > 0 {
- state = base.ReviewStateCommented
- content = review.Result.Comment
- }
- if review.Result.Approved {
- state = base.ReviewStateApproved
- }
- }
-
- poster := d.tryGetUser(review.UserID)
- reviews = append(reviews, &base.Review{
- IssueIndex: context.LocalID(),
- ReviewerID: poster.ID,
- ReviewerName: poster.Name,
- Content: content,
- State: state,
- })
- }
-
- return reviews, nil
-}
-
-// GetTopics return repository topics
-func (d *OneDevDownloader) GetTopics() ([]string, error) {
- return []string{}, nil
-}
-
-func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser {
- user, ok := d.userMap[userID]
- if !ok {
- err := d.callAPI(
- fmt.Sprintf("/api/users/%d", userID),
- nil,
- &user,
- )
- if err != nil {
- user = &onedevUser{
- Name: fmt.Sprintf("User %d", userID),
- }
- }
- d.userMap[userID] = user
- }
-
- return user
-}
+++ /dev/null
-// Copyright 2021 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "fmt"
- "net/http"
- "net/url"
- "testing"
- "time"
-
- "code.gitea.io/gitea/modules/migrations/base"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestOneDevDownloadRepo(t *testing.T) {
- resp, err := http.Get("https://code.onedev.io/projects/go-gitea-test_repo")
- if err != nil || resp.StatusCode != 200 {
- t.Skipf("Can't access test repo, skipping %s", t.Name())
- }
-
- u, _ := url.Parse("https://code.onedev.io")
- downloader := NewOneDevDownloader(context.Background(), u, "", "", "go-gitea-test_repo")
- if err != nil {
- t.Fatal(fmt.Sprintf("NewOneDevDownloader is nil: %v", err))
- }
- repo, err := downloader.GetRepoInfo()
- assert.NoError(t, err)
- assertRepositoryEqual(t, &base.Repository{
- Name: "go-gitea-test_repo",
- Owner: "",
- Description: "Test repository for testing migration from OneDev to gitea",
- CloneURL: "https://code.onedev.io/go-gitea-test_repo",
- OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo",
- }, repo)
-
- milestones, err := downloader.GetMilestones()
- assert.NoError(t, err)
- deadline := time.Unix(1620086400, 0)
- assertMilestonesEqual(t, []*base.Milestone{
- {
- Title: "1.0.0",
- Deadline: &deadline,
- Closed: &deadline,
- },
- {
- Title: "1.1.0",
- Description: "next things?",
- },
- }, milestones)
-
- labels, err := downloader.GetLabels()
- assert.NoError(t, err)
- assert.Len(t, labels, 6)
-
- issues, isEnd, err := downloader.GetIssues(1, 2)
- assert.NoError(t, err)
- assert.False(t, isEnd)
- assertIssuesEqual(t, []*base.Issue{
- {
- Number: 4,
- Title: "Hi there",
- Content: "an issue not assigned to a milestone",
- PosterName: "User 336",
- State: "open",
- Created: time.Unix(1628549776, 734000000),
- Updated: time.Unix(1628549776, 734000000),
- Labels: []*base.Label{
- {
- Name: "Improvement",
- },
- },
- Context: onedevIssueContext{
- foreignID: 398,
- localID: 4,
- IsPullRequest: false,
- },
- },
- {
- Number: 3,
- Title: "Add an awesome feature",
- Content: "just another issue to test against",
- PosterName: "User 336",
- State: "open",
- Milestone: "1.1.0",
- Created: time.Unix(1628549749, 878000000),
- Updated: time.Unix(1628549749, 878000000),
- Labels: []*base.Label{
- {
- Name: "New Feature",
- },
- },
- Context: onedevIssueContext{
- foreignID: 397,
- localID: 3,
- IsPullRequest: false,
- },
- },
- }, issues)
-
- comments, _, err := downloader.GetComments(base.GetCommentOptions{
- Context: onedevIssueContext{
- foreignID: 398,
- localID: 4,
- IsPullRequest: false,
- },
- })
- assert.NoError(t, err)
- assertCommentsEqual(t, []*base.Comment{
- {
- IssueIndex: 4,
- PosterName: "User 336",
- Created: time.Unix(1628549791, 128000000),
- Updated: time.Unix(1628549791, 128000000),
- Content: "it has a comment\n\nEDIT: that got edited",
- },
- }, comments)
-
- prs, _, err := downloader.GetPullRequests(1, 1)
- assert.NoError(t, err)
- assertPullRequestsEqual(t, []*base.PullRequest{
- {
- Number: 5,
- Title: "Pull to add a new file",
- Content: "just do some git stuff",
- PosterName: "User 336",
- State: "open",
- Created: time.Unix(1628550076, 25000000),
- Updated: time.Unix(1628550076, 25000000),
- Head: base.PullRequestBranch{
- Ref: "branch-for-a-pull",
- SHA: "343deffe3526b9bc84e873743ff7f6e6d8b827c0",
- RepoName: "go-gitea-test_repo",
- },
- Base: base.PullRequestBranch{
- Ref: "master",
- SHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
- RepoName: "go-gitea-test_repo",
- },
- Context: onedevIssueContext{
- foreignID: 186,
- localID: 5,
- IsPullRequest: true,
- },
- },
- }, prs)
-
- rvs, err := downloader.GetReviews(onedevIssueContext{
- foreignID: 186,
- localID: 5,
- })
- assert.NoError(t, err)
- assertReviewsEqual(t, []*base.Review{
- {
- IssueIndex: 5,
- ReviewerName: "User 317",
- State: "PENDING",
- },
- }, rvs)
-}
+++ /dev/null
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
- "fmt"
- "os"
- "path/filepath"
- "strconv"
-
- "code.gitea.io/gitea/modules/migrations/base"
-
- "gopkg.in/yaml.v2"
-)
-
-// RepositoryRestorer implements an Downloader from the local directory
-type RepositoryRestorer struct {
- base.NullDownloader
- ctx context.Context
- baseDir string
- repoOwner string
- repoName string
-}
-
-// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder
-func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) (*RepositoryRestorer, error) {
- baseDir, err := filepath.Abs(baseDir)
- if err != nil {
- return nil, err
- }
- return &RepositoryRestorer{
- ctx: ctx,
- baseDir: baseDir,
- repoOwner: owner,
- repoName: repoName,
- }, nil
-}
-
-func (r *RepositoryRestorer) commentDir() string {
- return filepath.Join(r.baseDir, "comments")
-}
-
-func (r *RepositoryRestorer) reviewDir() string {
- return filepath.Join(r.baseDir, "reviews")
-}
-
-// SetContext set context
-func (r *RepositoryRestorer) SetContext(ctx context.Context) {
- r.ctx = ctx
-}
-
-func (r *RepositoryRestorer) getRepoOptions() (map[string]string, error) {
- p := filepath.Join(r.baseDir, "repo.yml")
- bs, err := os.ReadFile(p)
- if err != nil {
- return nil, err
- }
-
- var opts = make(map[string]string)
- err = yaml.Unmarshal(bs, &opts)
- if err != nil {
- return nil, err
- }
- return opts, nil
-}
-
-// GetRepoInfo returns a repository information
-func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) {
- opts, err := r.getRepoOptions()
- if err != nil {
- return nil, err
- }
-
- isPrivate, _ := strconv.ParseBool(opts["is_private"])
-
- return &base.Repository{
- Owner: r.repoOwner,
- Name: r.repoName,
- IsPrivate: isPrivate,
- Description: opts["description"],
- OriginalURL: opts["original_url"],
- CloneURL: filepath.Join(r.baseDir, "git"),
- DefaultBranch: opts["default_branch"],
- }, nil
-}
-
-// GetTopics return github topics
-func (r *RepositoryRestorer) GetTopics() ([]string, error) {
- p := filepath.Join(r.baseDir, "topic.yml")
-
- var topics = struct {
- Topics []string `yaml:"topics"`
- }{}
-
- bs, err := os.ReadFile(p)
- if err != nil {
- return nil, err
- }
-
- err = yaml.Unmarshal(bs, &topics)
- if err != nil {
- return nil, err
- }
- return topics.Topics, nil
-}
-
-// GetMilestones returns milestones
-func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) {
- var milestones = make([]*base.Milestone, 0, 10)
- p := filepath.Join(r.baseDir, "milestone.yml")
- _, err := os.Stat(p)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, nil
- }
- return nil, err
- }
-
- bs, err := os.ReadFile(p)
- if err != nil {
- return nil, err
- }
-
- err = yaml.Unmarshal(bs, &milestones)
- if err != nil {
- return nil, err
- }
- return milestones, nil
-}
-
-// GetReleases returns releases
-func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) {
- var releases = make([]*base.Release, 0, 10)
- p := filepath.Join(r.baseDir, "release.yml")
- _, err := os.Stat(p)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, nil
- }
- return nil, err
- }
-
- bs, err := os.ReadFile(p)
- if err != nil {
- return nil, err
- }
-
- err = yaml.Unmarshal(bs, &releases)
- if err != nil {
- return nil, err
- }
- for _, rel := range releases {
- for _, asset := range rel.Assets {
- if asset.DownloadURL != nil {
- *asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL)
- }
- }
- }
- return releases, nil
-}
-
-// GetLabels returns labels
-func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) {
- var labels = make([]*base.Label, 0, 10)
- p := filepath.Join(r.baseDir, "label.yml")
- _, err := os.Stat(p)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, nil
- }
- return nil, err
- }
-
- bs, err := os.ReadFile(p)
- if err != nil {
- return nil, err
- }
-
- err = yaml.Unmarshal(bs, &labels)
- if err != nil {
- return nil, err
- }
- return labels, nil
-}
-
-// GetIssues returns issues according start and limit
-func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
- var issues = make([]*base.Issue, 0, 10)
- p := filepath.Join(r.baseDir, "issue.yml")
- _, err := os.Stat(p)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, true, nil
- }
- return nil, false, err
- }
-
- bs, err := os.ReadFile(p)
- if err != nil {
- return nil, false, err
- }
-
- err = yaml.Unmarshal(bs, &issues)
- if err != nil {
- return nil, false, err
- }
- for _, issue := range issues {
- issue.Context = base.BasicIssueContext(issue.Number)
- }
- return issues, true, nil
-}
-
-// GetComments returns comments according issueNumber
-func (r *RepositoryRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
- var comments = make([]*base.Comment, 0, 10)
- p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.Context.ForeignID()))
- _, err := os.Stat(p)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, false, nil
- }
- return nil, false, err
- }
-
- bs, err := os.ReadFile(p)
- if err != nil {
- return nil, false, err
- }
-
- err = yaml.Unmarshal(bs, &comments)
- if err != nil {
- return nil, false, err
- }
- return comments, false, nil
-}
-
-// GetPullRequests returns pull requests according page and perPage
-func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
- var pulls = make([]*base.PullRequest, 0, 10)
- p := filepath.Join(r.baseDir, "pull_request.yml")
- _, err := os.Stat(p)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, true, nil
- }
- return nil, false, err
- }
-
- bs, err := os.ReadFile(p)
- if err != nil {
- return nil, false, err
- }
-
- err = yaml.Unmarshal(bs, &pulls)
- if err != nil {
- return nil, false, err
- }
- for _, pr := range pulls {
- pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL)
- pr.Context = base.BasicIssueContext(pr.Number)
- }
- return pulls, true, nil
-}
-
-// GetReviews returns pull requests review
-func (r *RepositoryRestorer) GetReviews(context base.IssueContext) ([]*base.Review, error) {
- var reviews = make([]*base.Review, 0, 10)
- p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", context.ForeignID()))
- _, err := os.Stat(p)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, nil
- }
- return nil, err
- }
-
- bs, err := os.ReadFile(p)
- if err != nil {
- return nil, err
- }
-
- err = yaml.Unmarshal(bs, &reviews)
- if err != nil {
- return nil, err
- }
- return reviews, nil
-}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package migrations
-
-import (
- "context"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/models/db"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/structs"
-)
-
-// UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID
-func UpdateMigrationPosterID(ctx context.Context) error {
- for _, gitService := range structs.SupportedFullGitService {
- select {
- case <-ctx.Done():
- log.Warn("UpdateMigrationPosterID aborted before %s", gitService.Name())
- return db.ErrCancelledf("during UpdateMigrationPosterID before %s", gitService.Name())
- default:
- }
- if err := updateMigrationPosterIDByGitService(ctx, gitService); err != nil {
- log.Error("updateMigrationPosterIDByGitService failed: %v", err)
- }
- }
- return nil
-}
-
-func updateMigrationPosterIDByGitService(ctx context.Context, tp structs.GitServiceType) error {
- provider := tp.Name()
- if len(provider) == 0 {
- return nil
- }
-
- const batchSize = 100
- var start int
- for {
- select {
- case <-ctx.Done():
- log.Warn("UpdateMigrationPosterIDByGitService(%s) cancelled", tp.Name())
- return nil
- default:
- }
-
- users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{
- Provider: provider,
- Start: start,
- Limit: batchSize,
- })
- if err != nil {
- return err
- }
-
- for _, user := range users {
- select {
- case <-ctx.Done():
- log.Warn("UpdateMigrationPosterIDByGitService(%s) cancelled", tp.Name())
- return nil
- default:
- }
- externalUserID := user.ExternalID
- if err := models.UpdateMigrationsByType(tp, externalUserID, user.UserID); err != nil {
- log.Error("UpdateMigrationsByType type %s external user id %v to local user id %v failed: %v", tp.Name(), user.ExternalID, user.UserID, err)
- }
- }
-
- if len(users) < batchSize {
- break
- }
- start += len(users)
- }
- return nil
-}
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
- migration "code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations"
- migration "code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
+ "code.gitea.io/gitea/services/migrations"
)
func handleCreateError(owner *models.User, err error) error {
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations/base"
+ base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/queue"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/secret"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations"
- "code.gitea.io/gitea/modules/migrations/base"
+ base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/notification"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/services/forms"
+ "code.gitea.io/gitea/services/migrations"
)
// Migrate migrate remote git repository to gitea
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/external"
- repo_migrations "code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/notification"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/auth/source/oauth2"
"code.gitea.io/gitea/services/cron"
"code.gitea.io/gitea/services/mailer"
+ repo_migrations "code.gitea.io/gitea/services/migrations"
mirror_service "code.gitea.io/gitea/services/mirror"
pull_service "code.gitea.io/gitea/services/pull"
"code.gitea.io/gitea/services/repository"
myCtx "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/json"
- "code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/private"
+ "code.gitea.io/gitea/services/migrations"
)
// RestoreRepo restore a repository from data
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/task"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/services/forms"
+ "code.gitea.io/gitea/services/migrations"
)
const (
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/routers/utils"
"code.gitea.io/gitea/services/forms"
"code.gitea.io/gitea/services/mailer"
+ "code.gitea.io/gitea/services/migrations"
mirror_service "code.gitea.io/gitea/services/mirror"
repo_service "code.gitea.io/gitea/services/repository"
wiki_service "code.gitea.io/gitea/services/wiki"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/webhook"
- "code.gitea.io/gitea/modules/migrations"
repository_service "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/auth"
+ "code.gitea.io/gitea/services/migrations"
mirror_service "code.gitea.io/gitea/services/mirror"
)
--- /dev/null
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/log"
+ base "code.gitea.io/gitea/modules/migration"
+ "code.gitea.io/gitea/modules/repository"
+ "code.gitea.io/gitea/modules/structs"
+
+ "gopkg.in/yaml.v2"
+)
+
+var (
+ _ base.Uploader = &RepositoryDumper{}
+)
+
+// RepositoryDumper implements an Uploader to the local directory
+type RepositoryDumper struct {
+ ctx context.Context
+ baseDir string
+ repoOwner string
+ repoName string
+ opts base.MigrateOptions
+ milestoneFile *os.File
+ labelFile *os.File
+ releaseFile *os.File
+ issueFile *os.File
+ commentFiles map[int64]*os.File
+ pullrequestFile *os.File
+ reviewFiles map[int64]*os.File
+
+ gitRepo *git.Repository
+ prHeadCache map[string]struct{}
+}
+
+// NewRepositoryDumper creates an gitea Uploader
+func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) (*RepositoryDumper, error) {
+ baseDir = filepath.Join(baseDir, repoOwner, repoName)
+ if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
+ return nil, err
+ }
+ return &RepositoryDumper{
+ ctx: ctx,
+ opts: opts,
+ baseDir: baseDir,
+ repoOwner: repoOwner,
+ repoName: repoName,
+ prHeadCache: make(map[string]struct{}),
+ commentFiles: make(map[int64]*os.File),
+ reviewFiles: make(map[int64]*os.File),
+ }, nil
+}
+
+// MaxBatchInsertSize returns the table's max batch insert size
+func (g *RepositoryDumper) MaxBatchInsertSize(tp string) int {
+ return 1000
+}
+
+func (g *RepositoryDumper) gitPath() string {
+ return filepath.Join(g.baseDir, "git")
+}
+
+func (g *RepositoryDumper) wikiPath() string {
+ return filepath.Join(g.baseDir, "wiki")
+}
+
+func (g *RepositoryDumper) commentDir() string {
+ return filepath.Join(g.baseDir, "comments")
+}
+
+func (g *RepositoryDumper) reviewDir() string {
+ return filepath.Join(g.baseDir, "reviews")
+}
+
+func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
+ if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 {
+ u, err := url.Parse(remoteAddr)
+ if err != nil {
+ return "", err
+ }
+ u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword)
+ if len(g.opts.AuthToken) > 0 {
+ u.User = url.UserPassword("oauth2", g.opts.AuthToken)
+ }
+ remoteAddr = u.String()
+ }
+
+ return remoteAddr, nil
+}
+
+// CreateRepo creates a repository
+func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
+ f, err := os.Create(filepath.Join(g.baseDir, "repo.yml"))
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ bs, err := yaml.Marshal(map[string]interface{}{
+ "name": repo.Name,
+ "owner": repo.Owner,
+ "description": repo.Description,
+ "clone_addr": opts.CloneAddr,
+ "original_url": repo.OriginalURL,
+ "is_private": opts.Private,
+ "service_type": opts.GitServiceType,
+ "wiki": opts.Wiki,
+ "issues": opts.Issues,
+ "milestones": opts.Milestones,
+ "labels": opts.Labels,
+ "releases": opts.Releases,
+ "comments": opts.Comments,
+ "pulls": opts.PullRequests,
+ "assets": opts.ReleaseAssets,
+ })
+ if err != nil {
+ return err
+ }
+
+ if _, err := f.Write(bs); err != nil {
+ return err
+ }
+
+ repoPath := g.gitPath()
+ if err := os.MkdirAll(repoPath, os.ModePerm); err != nil {
+ return err
+ }
+
+ migrateTimeout := 2 * time.Hour
+
+ remoteAddr, err := g.setURLToken(repo.CloneURL)
+ if err != nil {
+ return err
+ }
+
+ err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{
+ Mirror: true,
+ Quiet: true,
+ Timeout: migrateTimeout,
+ })
+ if err != nil {
+ return fmt.Errorf("Clone: %v", err)
+ }
+
+ if opts.Wiki {
+ wikiPath := g.wikiPath()
+ wikiRemotePath := repository.WikiRemoteURL(remoteAddr)
+ if len(wikiRemotePath) > 0 {
+ if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil {
+ return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
+ }
+
+ if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
+ Mirror: true,
+ Quiet: true,
+ Timeout: migrateTimeout,
+ Branch: "master",
+ }); err != nil {
+ log.Warn("Clone wiki: %v", err)
+ if err := os.RemoveAll(wikiPath); err != nil {
+ return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
+ }
+ }
+ }
+ }
+
+ g.gitRepo, err = git.OpenRepository(g.gitPath())
+ return err
+}
+
+// Close closes this uploader
+func (g *RepositoryDumper) Close() {
+ if g.gitRepo != nil {
+ g.gitRepo.Close()
+ }
+ if g.milestoneFile != nil {
+ g.milestoneFile.Close()
+ }
+ if g.labelFile != nil {
+ g.labelFile.Close()
+ }
+ if g.releaseFile != nil {
+ g.releaseFile.Close()
+ }
+ if g.issueFile != nil {
+ g.issueFile.Close()
+ }
+ for _, f := range g.commentFiles {
+ f.Close()
+ }
+ if g.pullrequestFile != nil {
+ g.pullrequestFile.Close()
+ }
+ for _, f := range g.reviewFiles {
+ f.Close()
+ }
+}
+
+// CreateTopics creates topics
+func (g *RepositoryDumper) CreateTopics(topics ...string) error {
+ f, err := os.Create(filepath.Join(g.baseDir, "topic.yml"))
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ bs, err := yaml.Marshal(map[string]interface{}{
+ "topics": topics,
+ })
+ if err != nil {
+ return err
+ }
+
+ if _, err := f.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CreateMilestones creates milestones
+func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error {
+ var err error
+ if g.milestoneFile == nil {
+ g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml"))
+ if err != nil {
+ return err
+ }
+ }
+
+ bs, err := yaml.Marshal(milestones)
+ if err != nil {
+ return err
+ }
+
+ if _, err := g.milestoneFile.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CreateLabels creates labels
+func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error {
+ var err error
+ if g.labelFile == nil {
+ g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml"))
+ if err != nil {
+ return err
+ }
+ }
+
+ bs, err := yaml.Marshal(labels)
+ if err != nil {
+ return err
+ }
+
+ if _, err := g.labelFile.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CreateReleases creates releases
+func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error {
+ if g.opts.ReleaseAssets {
+ for _, release := range releases {
+ attachDir := filepath.Join("release_assets", release.TagName)
+ if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil {
+ return err
+ }
+ for _, asset := range release.Assets {
+ attachLocalPath := filepath.Join(attachDir, asset.Name)
+ // download attachment
+
+ err := func(attachPath string) error {
+ var rc io.ReadCloser
+ var err error
+ if asset.DownloadURL == nil {
+ rc, err = asset.DownloadFunc()
+ if err != nil {
+ return err
+ }
+ } else {
+ resp, err := http.Get(*asset.DownloadURL)
+ if err != nil {
+ return err
+ }
+ rc = resp.Body
+ }
+ defer rc.Close()
+
+ fw, err := os.Create(attachPath)
+ if err != nil {
+ return fmt.Errorf("Create: %v", err)
+ }
+ defer fw.Close()
+
+ _, err = io.Copy(fw, rc)
+ return err
+ }(filepath.Join(g.baseDir, attachLocalPath))
+ if err != nil {
+ return err
+ }
+ asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source
+ }
+ }
+ }
+
+ var err error
+ if g.releaseFile == nil {
+ g.releaseFile, err = os.Create(filepath.Join(g.baseDir, "release.yml"))
+ if err != nil {
+ return err
+ }
+ }
+
+ bs, err := yaml.Marshal(releases)
+ if err != nil {
+ return err
+ }
+
+ if _, err := g.releaseFile.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// SyncTags syncs releases with tags in the database
+func (g *RepositoryDumper) SyncTags() error {
+ return nil
+}
+
+// CreateIssues creates issues
+func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error {
+ var err error
+ if g.issueFile == nil {
+ g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml"))
+ if err != nil {
+ return err
+ }
+ }
+
+ bs, err := yaml.Marshal(issues)
+ if err != nil {
+ return err
+ }
+
+ if _, err := g.issueFile.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File, itemsMap map[int64][]interface{}) error {
+ if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+ return err
+ }
+
+ for number, items := range itemsMap {
+ var err error
+ itemFile := itemFiles[number]
+ if itemFile == nil {
+ itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number)))
+ if err != nil {
+ return err
+ }
+ itemFiles[number] = itemFile
+ }
+
+ bs, err := yaml.Marshal(items)
+ if err != nil {
+ return err
+ }
+
+ if _, err := itemFile.Write(bs); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// CreateComments creates comments of issues
+func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error {
+ var commentsMap = make(map[int64][]interface{}, len(comments))
+ for _, comment := range comments {
+ commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment)
+ }
+
+ return g.createItems(g.commentDir(), g.commentFiles, commentsMap)
+}
+
+// CreatePullRequests creates pull requests
+func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
+ for _, pr := range prs {
+ // download patch file
+ err := func() error {
+ u, err := g.setURLToken(pr.PatchURL)
+ if err != nil {
+ return err
+ }
+ resp, err := http.Get(u)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ pullDir := filepath.Join(g.gitPath(), "pulls")
+ if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
+ return err
+ }
+ fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))
+ f, err := os.Create(fPath)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ if _, err = io.Copy(f, resp.Body); err != nil {
+ return err
+ }
+ pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number)
+
+ return nil
+ }()
+ if err != nil {
+ return err
+ }
+
+ // set head information
+ pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
+ if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
+ return err
+ }
+ p, err := os.Create(filepath.Join(pullHead, "head"))
+ if err != nil {
+ return err
+ }
+ _, err = p.WriteString(pr.Head.SHA)
+ p.Close()
+ if err != nil {
+ return err
+ }
+
+ if pr.IsForkPullRequest() && pr.State != "closed" {
+ if pr.Head.OwnerName != "" {
+ remote := pr.Head.OwnerName
+ _, ok := g.prHeadCache[remote]
+ if !ok {
+ // git remote add
+ // TODO: how to handle private CloneURL?
+ err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
+ if err != nil {
+ log.Error("AddRemote failed: %s", err)
+ } else {
+ g.prHeadCache[remote] = struct{}{}
+ ok = true
+ }
+ }
+
+ if ok {
+ _, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.gitPath())
+ if err != nil {
+ log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
+ } else {
+ // a new branch name with <original_owner_name/original_branchname> will be created to as new head branch
+ ref := path.Join(pr.Head.OwnerName, pr.Head.Ref)
+ headBranch := filepath.Join(g.gitPath(), "refs", "heads", ref)
+ if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
+ return err
+ }
+ b, err := os.Create(headBranch)
+ if err != nil {
+ return err
+ }
+ _, err = b.WriteString(pr.Head.SHA)
+ b.Close()
+ if err != nil {
+ return err
+ }
+ pr.Head.Ref = ref
+ }
+ }
+ }
+ }
+ // whatever it's a forked repo PR, we have to change head info as the same as the base info
+ pr.Head.OwnerName = pr.Base.OwnerName
+ pr.Head.RepoName = pr.Base.RepoName
+ }
+
+ var err error
+ if g.pullrequestFile == nil {
+ if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil {
+ return err
+ }
+ g.pullrequestFile, err = os.Create(filepath.Join(g.baseDir, "pull_request.yml"))
+ if err != nil {
+ return err
+ }
+ }
+
+ bs, err := yaml.Marshal(prs)
+ if err != nil {
+ return err
+ }
+
+ if _, err := g.pullrequestFile.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CreateReviews create pull request reviews
+func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error {
+ var reviewsMap = make(map[int64][]interface{}, len(reviews))
+ for _, review := range reviews {
+ reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review)
+ }
+
+ return g.createItems(g.reviewDir(), g.reviewFiles, reviewsMap)
+}
+
+// Rollback when migrating failed, this will rollback all the changes.
+func (g *RepositoryDumper) Rollback() error {
+ g.Close()
+ return os.RemoveAll(g.baseDir)
+}
+
+// Finish when migrating succeed, this will update something.
+func (g *RepositoryDumper) Finish() error {
+ return nil
+}
+
+// DumpRepository dump repository according MigrateOptions to a local directory
+func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error {
+ downloader, err := newDownloader(ctx, ownerName, opts)
+ if err != nil {
+ return err
+ }
+ uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts)
+ if err != nil {
+ return err
+ }
+
+ if err := migrateRepository(downloader, uploader, opts, nil); err != nil {
+ if err1 := uploader.Rollback(); err1 != nil {
+ log.Error("rollback failed: %v", err1)
+ }
+ return err
+ }
+ return nil
+}
+
+func updateOptionsUnits(opts *base.MigrateOptions, units []string) {
+ if len(units) == 0 {
+ opts.Wiki = true
+ opts.Issues = true
+ opts.Milestones = true
+ opts.Labels = true
+ opts.Releases = true
+ opts.Comments = true
+ opts.PullRequests = true
+ opts.ReleaseAssets = true
+ } else {
+ for _, unit := range units {
+ switch strings.ToLower(unit) {
+ case "wiki":
+ opts.Wiki = true
+ case "issues":
+ opts.Issues = true
+ case "milestones":
+ opts.Milestones = true
+ case "labels":
+ opts.Labels = true
+ case "releases":
+ opts.Releases = true
+ case "release_assets":
+ opts.ReleaseAssets = true
+ case "comments":
+ opts.Comments = true
+ case "pull_requests":
+ opts.PullRequests = true
+ }
+ }
+ }
+}
+
+// RestoreRepository restore a repository from the disk directory
+func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string, units []string) error {
+ doer, err := models.GetAdminUser()
+ if err != nil {
+ return err
+ }
+ var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName)
+ downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName)
+ if err != nil {
+ return err
+ }
+ opts, err := downloader.getRepoOptions()
+ if err != nil {
+ return err
+ }
+ tp, _ := strconv.Atoi(opts["service_type"])
+
+ var migrateOpts = base.MigrateOptions{
+ GitServiceType: structs.GitServiceType(tp),
+ }
+ updateOptionsUnits(&migrateOpts, units)
+
+ if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil {
+ if err1 := uploader.Rollback(); err1 != nil {
+ log.Error("rollback failed: %v", err1)
+ }
+ return err
+ }
+ return updateMigrationPosterIDByGitService(ctx, structs.GitServiceType(tp))
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "errors"
+
+ "github.com/google/go-github/v39/github"
+)
+
+var (
+ // ErrRepoNotCreated returns the error that repository not created
+ ErrRepoNotCreated = errors.New("repository is not created yet")
+)
+
+// IsRateLimitError returns true if the err is github.RateLimitError
+func IsRateLimitError(err error) bool {
+ _, ok := err.(*github.RateLimitError)
+ return ok
+}
+
+// IsTwoFactorAuthError returns true if the err is github.TwoFactorAuthError
+func IsTwoFactorAuthError(err error) bool {
+ _, ok := err.(*github.TwoFactorAuthError)
+ return ok
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+
+ base "code.gitea.io/gitea/modules/migration"
+)
+
+var (
+ _ base.Downloader = &PlainGitDownloader{}
+)
+
+// PlainGitDownloader implements a Downloader interface to clone git from a http/https URL
+type PlainGitDownloader struct {
+ base.NullDownloader
+ ownerName string
+ repoName string
+ remoteURL string
+}
+
+// NewPlainGitDownloader creates a git Downloader
+func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownloader {
+ return &PlainGitDownloader{
+ ownerName: ownerName,
+ repoName: repoName,
+ remoteURL: remoteURL,
+ }
+}
+
+// SetContext set context
+func (g *PlainGitDownloader) SetContext(ctx context.Context) {
+}
+
+// GetRepoInfo returns a repository information
+func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
+ // convert github repo to stand Repo
+ return &base.Repository{
+ Owner: g.ownerName,
+ Name: g.repoName,
+ CloneURL: g.remoteURL,
+ }, nil
+}
+
+// GetTopics return empty string slice
+func (g PlainGitDownloader) GetTopics() ([]string, error) {
+ return []string{}, nil
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "net/url"
+ "strings"
+
+ base "code.gitea.io/gitea/modules/migration"
+ "code.gitea.io/gitea/modules/structs"
+)
+
+var (
+ _ base.Downloader = &GitBucketDownloader{}
+ _ base.DownloaderFactory = &GitBucketDownloaderFactory{}
+)
+
+func init() {
+ RegisterDownloaderFactory(&GitBucketDownloaderFactory{})
+}
+
+// GitBucketDownloaderFactory defines a GitBucket downloader factory
+type GitBucketDownloaderFactory struct {
+}
+
+// New returns a Downloader related to this factory according MigrateOptions
+func (f *GitBucketDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
+ u, err := url.Parse(opts.CloneAddr)
+ if err != nil {
+ return nil, err
+ }
+
+ baseURL := u.Scheme + "://" + u.Host
+ fields := strings.Split(u.Path, "/")
+ oldOwner := fields[1]
+ oldName := strings.TrimSuffix(fields[2], ".git")
+
+ return NewGitBucketDownloader(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
+}
+
+// GitServiceType returns the type of git service
+func (f *GitBucketDownloaderFactory) GitServiceType() structs.GitServiceType {
+ return structs.GitBucketService
+}
+
+// GitBucketDownloader implements a Downloader interface to get repository information
+// from GitBucket via GithubDownloader
+type GitBucketDownloader struct {
+ *GithubDownloaderV3
+}
+
+// NewGitBucketDownloader creates a GitBucket downloader
+func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GitBucketDownloader {
+ githubDownloader := NewGithubDownloaderV3(ctx, baseURL, userName, password, token, repoOwner, repoName)
+ githubDownloader.SkipReactions = true
+ return &GitBucketDownloader{
+ githubDownloader,
+ }
+}
+
+// SupportGetRepoComments return true if it supports get repo comments
+func (g *GitBucketDownloader) SupportGetRepoComments() bool {
+ return false
+}
+
+// GetReviews is not supported
+func (g *GitBucketDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
+ return nil, &base.ErrNotSupported{Entity: "Reviews"}
+}
--- /dev/null
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/log"
+ base "code.gitea.io/gitea/modules/migration"
+ "code.gitea.io/gitea/modules/proxy"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/structs"
+
+ gitea_sdk "code.gitea.io/sdk/gitea"
+)
+
+var (
+ _ base.Downloader = &GiteaDownloader{}
+ _ base.DownloaderFactory = &GiteaDownloaderFactory{}
+)
+
+func init() {
+ RegisterDownloaderFactory(&GiteaDownloaderFactory{})
+}
+
+// GiteaDownloaderFactory defines a gitea downloader factory
+type GiteaDownloaderFactory struct {
+}
+
+// New returns a Downloader related to this factory according MigrateOptions
+func (f *GiteaDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
+ u, err := url.Parse(opts.CloneAddr)
+ if err != nil {
+ return nil, err
+ }
+
+ baseURL := u.Scheme + "://" + u.Host
+ repoNameSpace := strings.TrimPrefix(u.Path, "/")
+ repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
+
+ path := strings.Split(repoNameSpace, "/")
+ if len(path) < 2 {
+ return nil, fmt.Errorf("invalid path: %s", repoNameSpace)
+ }
+
+ repoPath := strings.Join(path[len(path)-2:], "/")
+ if len(path) > 2 {
+ subPath := strings.Join(path[:len(path)-2], "/")
+ baseURL += "/" + subPath
+ }
+
+ log.Trace("Create gitea downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
+
+ return NewGiteaDownloader(ctx, baseURL, repoPath, opts.AuthUsername, opts.AuthPassword, opts.AuthToken)
+}
+
+// GitServiceType returns the type of git service
+func (f *GiteaDownloaderFactory) GitServiceType() structs.GitServiceType {
+ return structs.GiteaService
+}
+
+// GiteaDownloader implements a Downloader interface to get repository information's
+type GiteaDownloader struct {
+ base.NullDownloader
+ ctx context.Context
+ client *gitea_sdk.Client
+ repoOwner string
+ repoName string
+ pagination bool
+ maxPerPage int
+}
+
+// NewGiteaDownloader creates a gitea Downloader via gitea API
+// Use either a username/password or personal token. token is preferred
+// Note: Public access only allows very basic access
+func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GiteaDownloader, error) {
+ giteaClient, err := gitea_sdk.NewClient(
+ baseURL,
+ gitea_sdk.SetToken(token),
+ gitea_sdk.SetBasicAuth(username, password),
+ gitea_sdk.SetContext(ctx),
+ gitea_sdk.SetHTTPClient(&http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }),
+ )
+ if err != nil {
+ log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
+ return nil, err
+ }
+
+ path := strings.Split(repoPath, "/")
+
+ paginationSupport := true
+ if err = giteaClient.CheckServerVersionConstraint(">=1.12"); err != nil {
+ paginationSupport = false
+ }
+
+ // set small maxPerPage since we can only guess
+ // (default would be 50 but this can differ)
+ maxPerPage := 10
+ // gitea instances >=1.13 can tell us what maximum they have
+ apiConf, _, err := giteaClient.GetGlobalAPISettings()
+ if err != nil {
+ log.Info("Unable to get global API settings. Ignoring these.")
+ log.Debug("giteaClient.GetGlobalAPISettings. Error: %v", err)
+ }
+ if apiConf != nil {
+ maxPerPage = apiConf.MaxResponseItems
+ }
+
+ return &GiteaDownloader{
+ ctx: ctx,
+ client: giteaClient,
+ repoOwner: path[0],
+ repoName: path[1],
+ pagination: paginationSupport,
+ maxPerPage: maxPerPage,
+ }, nil
+}
+
+// SetContext set context
+func (g *GiteaDownloader) SetContext(ctx context.Context) {
+ g.ctx = ctx
+}
+
+// GetRepoInfo returns a repository information
+func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) {
+ if g == nil {
+ return nil, errors.New("error: GiteaDownloader is nil")
+ }
+
+ repo, _, err := g.client.GetRepo(g.repoOwner, g.repoName)
+ if err != nil {
+ return nil, err
+ }
+
+ return &base.Repository{
+ Name: repo.Name,
+ Owner: repo.Owner.UserName,
+ IsPrivate: repo.Private,
+ Description: repo.Description,
+ CloneURL: repo.CloneURL,
+ OriginalURL: repo.HTMLURL,
+ DefaultBranch: repo.DefaultBranch,
+ }, nil
+}
+
+// GetTopics return gitea topics
+func (g *GiteaDownloader) GetTopics() ([]string, error) {
+ topics, _, err := g.client.ListRepoTopics(g.repoOwner, g.repoName, gitea_sdk.ListRepoTopicsOptions{})
+ return topics, err
+}
+
+// GetMilestones returns milestones
+func (g *GiteaDownloader) GetMilestones() ([]*base.Milestone, error) {
+ var milestones = make([]*base.Milestone, 0, g.maxPerPage)
+
+ for i := 1; ; i++ {
+ // make sure gitea can shutdown gracefully
+ select {
+ case <-g.ctx.Done():
+ return nil, nil
+ default:
+ }
+
+ ms, _, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName, gitea_sdk.ListMilestoneOption{
+ ListOptions: gitea_sdk.ListOptions{
+ PageSize: g.maxPerPage,
+ Page: i,
+ },
+ State: gitea_sdk.StateAll,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ for i := range ms {
+ // old gitea instances dont have this information
+ createdAT := time.Time{}
+ var updatedAT *time.Time
+ if ms[i].Closed != nil {
+ createdAT = *ms[i].Closed
+ updatedAT = ms[i].Closed
+ }
+
+ // new gitea instances (>=1.13) do
+ if !ms[i].Created.IsZero() {
+ createdAT = ms[i].Created
+ }
+ if ms[i].Updated != nil && !ms[i].Updated.IsZero() {
+ updatedAT = ms[i].Updated
+ }
+
+ milestones = append(milestones, &base.Milestone{
+ Title: ms[i].Title,
+ Description: ms[i].Description,
+ Deadline: ms[i].Deadline,
+ Created: createdAT,
+ Updated: updatedAT,
+ Closed: ms[i].Closed,
+ State: string(ms[i].State),
+ })
+ }
+ if !g.pagination || len(ms) < g.maxPerPage {
+ break
+ }
+ }
+ return milestones, nil
+}
+
+func (g *GiteaDownloader) convertGiteaLabel(label *gitea_sdk.Label) *base.Label {
+ return &base.Label{
+ Name: label.Name,
+ Color: label.Color,
+ Description: label.Description,
+ }
+}
+
+// GetLabels returns labels
+func (g *GiteaDownloader) GetLabels() ([]*base.Label, error) {
+ var labels = make([]*base.Label, 0, g.maxPerPage)
+
+ for i := 1; ; i++ {
+ // make sure gitea can shutdown gracefully
+ select {
+ case <-g.ctx.Done():
+ return nil, nil
+ default:
+ }
+
+ ls, _, err := g.client.ListRepoLabels(g.repoOwner, g.repoName, gitea_sdk.ListLabelsOptions{ListOptions: gitea_sdk.ListOptions{
+ PageSize: g.maxPerPage,
+ Page: i,
+ }})
+ if err != nil {
+ return nil, err
+ }
+
+ for i := range ls {
+ labels = append(labels, g.convertGiteaLabel(ls[i]))
+ }
+ if !g.pagination || len(ls) < g.maxPerPage {
+ break
+ }
+ }
+ return labels, nil
+}
+
+func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Release {
+ r := &base.Release{
+ TagName: rel.TagName,
+ TargetCommitish: rel.Target,
+ Name: rel.Title,
+ Body: rel.Note,
+ Draft: rel.IsDraft,
+ Prerelease: rel.IsPrerelease,
+ PublisherID: rel.Publisher.ID,
+ PublisherName: rel.Publisher.UserName,
+ PublisherEmail: rel.Publisher.Email,
+ Published: rel.PublishedAt,
+ Created: rel.CreatedAt,
+ }
+
+ httpClient := &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }
+
+ for _, asset := range rel.Attachments {
+ size := int(asset.Size)
+ dlCount := int(asset.DownloadCount)
+ r.Assets = append(r.Assets, &base.ReleaseAsset{
+ ID: asset.ID,
+ Name: asset.Name,
+ Size: &size,
+ DownloadCount: &dlCount,
+ Created: asset.Created,
+ DownloadURL: &asset.DownloadURL,
+ DownloadFunc: func() (io.ReadCloser, error) {
+ asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, rel.ID, asset.ID)
+ if err != nil {
+ return nil, err
+ }
+ // FIXME: for a private download?
+ req, err := http.NewRequest("GET", asset.DownloadURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ // resp.Body is closed by the uploader
+ return resp.Body, nil
+ },
+ })
+ }
+ return r
+}
+
+// GetReleases returns releases
+func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) {
+ var releases = make([]*base.Release, 0, g.maxPerPage)
+
+ for i := 1; ; i++ {
+ // make sure gitea can shutdown gracefully
+ select {
+ case <-g.ctx.Done():
+ return nil, nil
+ default:
+ }
+
+ rl, _, err := g.client.ListReleases(g.repoOwner, g.repoName, gitea_sdk.ListReleasesOptions{ListOptions: gitea_sdk.ListOptions{
+ PageSize: g.maxPerPage,
+ Page: i,
+ }})
+ if err != nil {
+ return nil, err
+ }
+
+ for i := range rl {
+ releases = append(releases, g.convertGiteaRelease(rl[i]))
+ }
+ if !g.pagination || len(rl) < g.maxPerPage {
+ break
+ }
+ }
+ return releases, nil
+}
+
+func (g *GiteaDownloader) getIssueReactions(index int64) ([]*base.Reaction, error) {
+ var reactions []*base.Reaction
+ if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
+ log.Info("GiteaDownloader: instance to old, skip getIssueReactions")
+ return reactions, nil
+ }
+ rl, _, err := g.client.GetIssueReactions(g.repoOwner, g.repoName, index)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, reaction := range rl {
+ reactions = append(reactions, &base.Reaction{
+ UserID: reaction.User.ID,
+ UserName: reaction.User.UserName,
+ Content: reaction.Reaction,
+ })
+ }
+ return reactions, nil
+}
+
+func (g *GiteaDownloader) getCommentReactions(commentID int64) ([]*base.Reaction, error) {
+ var reactions []*base.Reaction
+ if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
+ log.Info("GiteaDownloader: instance to old, skip getCommentReactions")
+ return reactions, nil
+ }
+ rl, _, err := g.client.GetIssueCommentReactions(g.repoOwner, g.repoName, commentID)
+ if err != nil {
+ return nil, err
+ }
+
+ for i := range rl {
+ reactions = append(reactions, &base.Reaction{
+ UserID: rl[i].User.ID,
+ UserName: rl[i].User.UserName,
+ Content: rl[i].Reaction,
+ })
+ }
+ return reactions, nil
+}
+
+// GetIssues returns issues according start and limit
+func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
+ if perPage > g.maxPerPage {
+ perPage = g.maxPerPage
+ }
+ var allIssues = make([]*base.Issue, 0, perPage)
+
+ issues, _, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gitea_sdk.ListIssueOption{
+ ListOptions: gitea_sdk.ListOptions{Page: page, PageSize: perPage},
+ State: gitea_sdk.StateAll,
+ Type: gitea_sdk.IssueTypeIssue,
+ })
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing issues: %v", err)
+ }
+ for _, issue := range issues {
+
+ var labels = make([]*base.Label, 0, len(issue.Labels))
+ for i := range issue.Labels {
+ labels = append(labels, g.convertGiteaLabel(issue.Labels[i]))
+ }
+
+ var milestone string
+ if issue.Milestone != nil {
+ milestone = issue.Milestone.Title
+ }
+
+ reactions, err := g.getIssueReactions(issue.Index)
+ if err != nil {
+ log.Warn("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)
+ if err2 := models.CreateRepositoryNotice(
+ fmt.Sprintf("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)); err2 != nil {
+ log.Error("create repository notice failed: ", err2)
+ }
+ }
+
+ var assignees []string
+ for i := range issue.Assignees {
+ assignees = append(assignees, issue.Assignees[i].UserName)
+ }
+
+ allIssues = append(allIssues, &base.Issue{
+ Title: issue.Title,
+ Number: issue.Index,
+ PosterID: issue.Poster.ID,
+ PosterName: issue.Poster.UserName,
+ PosterEmail: issue.Poster.Email,
+ Content: issue.Body,
+ Milestone: milestone,
+ State: string(issue.State),
+ Created: issue.Created,
+ Updated: issue.Updated,
+ Closed: issue.Closed,
+ Reactions: reactions,
+ Labels: labels,
+ Assignees: assignees,
+ IsLocked: issue.IsLocked,
+ Context: base.BasicIssueContext(issue.Index),
+ })
+ }
+
+ isEnd := len(issues) < perPage
+ if !g.pagination {
+ isEnd = len(issues) == 0
+ }
+ return allIssues, isEnd, nil
+}
+
+// GetComments returns comments according issueNumber
+func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
+ var allComments = make([]*base.Comment, 0, g.maxPerPage)
+
+ for i := 1; ; i++ {
+ // make sure gitea can shutdown gracefully
+ select {
+ case <-g.ctx.Done():
+ return nil, false, nil
+ default:
+ }
+
+ comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{
+ PageSize: g.maxPerPage,
+ Page: i,
+ }})
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.Context.ForeignID(), err)
+ }
+
+ for _, comment := range comments {
+ reactions, err := g.getCommentReactions(comment.ID)
+ if err != nil {
+ log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)
+ if err2 := models.CreateRepositoryNotice(
+ fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
+ log.Error("create repository notice failed: ", err2)
+ }
+ }
+
+ allComments = append(allComments, &base.Comment{
+ IssueIndex: opts.Context.LocalID(),
+ PosterID: comment.Poster.ID,
+ PosterName: comment.Poster.UserName,
+ PosterEmail: comment.Poster.Email,
+ Content: comment.Body,
+ Created: comment.Created,
+ Updated: comment.Updated,
+ Reactions: reactions,
+ })
+ }
+
+ if !g.pagination || len(comments) < g.maxPerPage {
+ break
+ }
+ }
+ return allComments, true, nil
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
+ if perPage > g.maxPerPage {
+ perPage = g.maxPerPage
+ }
+ var allPRs = make([]*base.PullRequest, 0, perPage)
+
+ prs, _, err := g.client.ListRepoPullRequests(g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
+ ListOptions: gitea_sdk.ListOptions{
+ Page: page,
+ PageSize: perPage,
+ },
+ State: gitea_sdk.StateAll,
+ })
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %v", page, perPage, err)
+ }
+ for _, pr := range prs {
+ var milestone string
+ if pr.Milestone != nil {
+ milestone = pr.Milestone.Title
+ }
+
+ var labels = make([]*base.Label, 0, len(pr.Labels))
+ for i := range pr.Labels {
+ labels = append(labels, g.convertGiteaLabel(pr.Labels[i]))
+ }
+
+ var (
+ headUserName string
+ headRepoName string
+ headCloneURL string
+ headRef string
+ headSHA string
+ )
+ if pr.Head != nil {
+ if pr.Head.Repository != nil {
+ headUserName = pr.Head.Repository.Owner.UserName
+ headRepoName = pr.Head.Repository.Name
+ headCloneURL = pr.Head.Repository.CloneURL
+ }
+ headSHA = pr.Head.Sha
+ headRef = pr.Head.Ref
+ }
+
+ var mergeCommitSHA string
+ if pr.MergedCommitID != nil {
+ mergeCommitSHA = *pr.MergedCommitID
+ }
+
+ reactions, err := g.getIssueReactions(pr.Index)
+ if err != nil {
+ log.Warn("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)
+ if err2 := models.CreateRepositoryNotice(
+ fmt.Sprintf("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)); err2 != nil {
+ log.Error("create repository notice failed: ", err2)
+ }
+ }
+
+ var assignees []string
+ for i := range pr.Assignees {
+ assignees = append(assignees, pr.Assignees[i].UserName)
+ }
+
+ createdAt := time.Time{}
+ if pr.Created != nil {
+ createdAt = *pr.Created
+ }
+ updatedAt := time.Time{}
+ if pr.Created != nil {
+ updatedAt = *pr.Updated
+ }
+
+ closedAt := pr.Closed
+ if pr.Merged != nil && closedAt == nil {
+ closedAt = pr.Merged
+ }
+
+ allPRs = append(allPRs, &base.PullRequest{
+ Title: pr.Title,
+ Number: pr.Index,
+ PosterID: pr.Poster.ID,
+ PosterName: pr.Poster.UserName,
+ PosterEmail: pr.Poster.Email,
+ Content: pr.Body,
+ State: string(pr.State),
+ Created: createdAt,
+ Updated: updatedAt,
+ Closed: closedAt,
+ Labels: labels,
+ Milestone: milestone,
+ Reactions: reactions,
+ Assignees: assignees,
+ Merged: pr.HasMerged,
+ MergedTime: pr.Merged,
+ MergeCommitSHA: mergeCommitSHA,
+ IsLocked: pr.IsLocked,
+ PatchURL: pr.PatchURL,
+ Head: base.PullRequestBranch{
+ Ref: headRef,
+ SHA: headSHA,
+ RepoName: headRepoName,
+ OwnerName: headUserName,
+ CloneURL: headCloneURL,
+ },
+ Base: base.PullRequestBranch{
+ Ref: pr.Base.Ref,
+ SHA: pr.Base.Sha,
+ RepoName: g.repoName,
+ OwnerName: g.repoOwner,
+ },
+ Context: base.BasicIssueContext(pr.Index),
+ })
+ }
+
+ isEnd := len(prs) < perPage
+ if !g.pagination {
+ isEnd = len(prs) == 0
+ }
+ return allPRs, isEnd, nil
+}
+
+// GetReviews returns pull requests review
+func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
+ if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil {
+ log.Info("GiteaDownloader: instance to old, skip GetReviews")
+ return nil, nil
+ }
+
+ var allReviews = make([]*base.Review, 0, g.maxPerPage)
+
+ for i := 1; ; i++ {
+ // make sure gitea can shutdown gracefully
+ select {
+ case <-g.ctx.Done():
+ return nil, nil
+ default:
+ }
+
+ prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, context.ForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{
+ Page: i,
+ PageSize: g.maxPerPage,
+ }})
+ if err != nil {
+ return nil, err
+ }
+
+ for _, pr := range prl {
+
+ rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, context.ForeignID(), pr.ID)
+ if err != nil {
+ return nil, err
+ }
+ var reviewComments []*base.ReviewComment
+ for i := range rcl {
+ line := int(rcl[i].LineNum)
+ if rcl[i].OldLineNum > 0 {
+ line = int(rcl[i].OldLineNum) * -1
+ }
+
+ reviewComments = append(reviewComments, &base.ReviewComment{
+ ID: rcl[i].ID,
+ Content: rcl[i].Body,
+ TreePath: rcl[i].Path,
+ DiffHunk: rcl[i].DiffHunk,
+ Line: line,
+ CommitID: rcl[i].CommitID,
+ PosterID: rcl[i].Reviewer.ID,
+ CreatedAt: rcl[i].Created,
+ UpdatedAt: rcl[i].Updated,
+ })
+ }
+
+ allReviews = append(allReviews, &base.Review{
+ ID: pr.ID,
+ IssueIndex: context.LocalID(),
+ ReviewerID: pr.Reviewer.ID,
+ ReviewerName: pr.Reviewer.UserName,
+ Official: pr.Official,
+ CommitID: pr.CommitID,
+ Content: pr.Body,
+ CreatedAt: pr.Submitted,
+ State: string(pr.State),
+ Comments: reviewComments,
+ })
+ }
+
+ if len(prl) < g.maxPerPage {
+ break
+ }
+ }
+ return allReviews, nil
+}
--- /dev/null
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "net/http"
+ "os"
+ "sort"
+ "testing"
+ "time"
+
+ base "code.gitea.io/gitea/modules/migration"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGiteaDownloadRepo(t *testing.T) {
+ // Skip tests if Gitea token is not found
+ giteaToken := os.Getenv("GITEA_TOKEN")
+ if giteaToken == "" {
+ t.Skip("skipped test because GITEA_TOKEN was not in the environment")
+ }
+
+ resp, err := http.Get("https://gitea.com/gitea")
+ if err != nil || resp.StatusCode != 200 {
+ t.Skipf("Can't reach https://gitea.com, skipping %s", t.Name())
+ }
+
+ downloader, err := NewGiteaDownloader(context.Background(), "https://gitea.com", "gitea/test_repo", "", "", giteaToken)
+ if downloader == nil {
+ t.Fatal("NewGitlabDownloader is nil")
+ }
+ if !assert.NoError(t, err) {
+ t.Fatal("NewGitlabDownloader error occur")
+ }
+
+ repo, err := downloader.GetRepoInfo()
+ assert.NoError(t, err)
+ assertRepositoryEqual(t, &base.Repository{
+ Name: "test_repo",
+ Owner: "gitea",
+ IsPrivate: false,
+ Description: "Test repository for testing migration from gitea to gitea",
+ CloneURL: "https://gitea.com/gitea/test_repo.git",
+ OriginalURL: "https://gitea.com/gitea/test_repo",
+ DefaultBranch: "master",
+ }, repo)
+
+ topics, err := downloader.GetTopics()
+ assert.NoError(t, err)
+ sort.Strings(topics)
+ assert.EqualValues(t, []string{"ci", "gitea", "migration", "test"}, topics)
+
+ labels, err := downloader.GetLabels()
+ assert.NoError(t, err)
+ assertLabelsEqual(t, []*base.Label{
+ {
+ Name: "Bug",
+ Color: "e11d21",
+ },
+ {
+ Name: "Enhancement",
+ Color: "207de5",
+ },
+ {
+ Name: "Feature",
+ Color: "0052cc",
+ Description: "a feature request",
+ },
+ {
+ Name: "Invalid",
+ Color: "d4c5f9",
+ },
+ {
+ Name: "Question",
+ Color: "fbca04",
+ },
+ {
+ Name: "Valid",
+ Color: "53e917",
+ },
+ }, labels)
+
+ milestones, err := downloader.GetMilestones()
+ assert.NoError(t, err)
+ assertMilestonesEqual(t, []*base.Milestone{
+ {
+ Title: "V2 Finalize",
+ Created: time.Unix(0, 0),
+ Deadline: timePtr(time.Unix(1599263999, 0)),
+ Updated: timePtr(time.Unix(0, 0)),
+ State: "open",
+ },
+ {
+ Title: "V1",
+ Description: "Generate Content",
+ Created: time.Unix(0, 0),
+ Updated: timePtr(time.Unix(0, 0)),
+ Closed: timePtr(time.Unix(1598985406, 0)),
+ State: "closed",
+ },
+ }, milestones)
+
+ releases, err := downloader.GetReleases()
+ assert.NoError(t, err)
+ assertReleasesEqual(t, []*base.Release{
+ {
+ Name: "Second Release",
+ TagName: "v2-rc1",
+ TargetCommitish: "master",
+ Body: "this repo has:\r\n* reactions\r\n* wiki\r\n* issues (open/closed)\r\n* pulls (open/closed/merged) (external/internal)\r\n* pull reviews\r\n* projects\r\n* milestones\r\n* labels\r\n* releases\r\n\r\nto test migration against",
+ Draft: false,
+ Prerelease: true,
+ Created: time.Date(2020, 9, 1, 18, 2, 43, 0, time.UTC),
+ Published: time.Date(2020, 9, 1, 18, 2, 43, 0, time.UTC),
+ PublisherID: 689,
+ PublisherName: "6543",
+ PublisherEmail: "6543@obermui.de",
+ },
+ {
+ Name: "First Release",
+ TagName: "V1",
+ TargetCommitish: "master",
+ Body: "as title",
+ Draft: false,
+ Prerelease: false,
+ Created: time.Date(2020, 9, 1, 17, 30, 32, 0, time.UTC),
+ Published: time.Date(2020, 9, 1, 17, 30, 32, 0, time.UTC),
+ PublisherID: 689,
+ PublisherName: "6543",
+ PublisherEmail: "6543@obermui.de",
+ },
+ }, releases)
+
+ issues, isEnd, err := downloader.GetIssues(1, 50)
+ assert.NoError(t, err)
+ assert.True(t, isEnd)
+ assert.Len(t, issues, 7)
+ assert.EqualValues(t, "open", issues[0].State)
+
+ issues, isEnd, err = downloader.GetIssues(3, 2)
+ assert.NoError(t, err)
+ assert.False(t, isEnd)
+
+ assertIssuesEqual(t, []*base.Issue{
+ {
+ Number: 4,
+ Title: "what is this repo about?",
+ Content: "",
+ Milestone: "V1",
+ PosterID: -1,
+ PosterName: "Ghost",
+ PosterEmail: "",
+ State: "closed",
+ IsLocked: true,
+ Created: time.Unix(1598975321, 0),
+ Updated: time.Unix(1598975400, 0),
+ Labels: []*base.Label{{
+ Name: "Question",
+ Color: "fbca04",
+ Description: "",
+ }},
+ Reactions: []*base.Reaction{
+ {
+ UserID: 689,
+ UserName: "6543",
+ Content: "gitea",
+ },
+ {
+ UserID: 689,
+ UserName: "6543",
+ Content: "laugh",
+ },
+ },
+ Closed: timePtr(time.Date(2020, 9, 1, 15, 49, 34, 0, time.UTC)),
+ },
+ {
+ Number: 2,
+ Title: "Spam",
+ Content: ":(",
+ Milestone: "",
+ PosterID: 689,
+ PosterName: "6543",
+ PosterEmail: "6543@obermui.de",
+ State: "closed",
+ IsLocked: false,
+ Created: time.Unix(1598919780, 0),
+ Updated: time.Unix(1598969497, 0),
+ Labels: []*base.Label{{
+ Name: "Invalid",
+ Color: "d4c5f9",
+ Description: "",
+ }},
+ Closed: timePtr(time.Unix(1598969497, 0)),
+ },
+ }, issues)
+
+ comments, _, err := downloader.GetComments(base.GetCommentOptions{
+ Context: base.BasicIssueContext(4),
+ })
+ assert.NoError(t, err)
+ assertCommentsEqual(t, []*base.Comment{
+ {
+ IssueIndex: 4,
+ PosterID: 689,
+ PosterName: "6543",
+ PosterEmail: "6543@obermui.de",
+ Created: time.Unix(1598975370, 0),
+ Updated: time.Unix(1599070865, 0),
+ Content: "a really good question!\n\nIt is the used as TESTSET for gitea2gitea repo migration function",
+ },
+ {
+ IssueIndex: 4,
+ PosterID: -1,
+ PosterName: "Ghost",
+ PosterEmail: "",
+ Created: time.Unix(1598975393, 0),
+ Updated: time.Unix(1598975393, 0),
+ Content: "Oh!",
+ },
+ }, comments)
+
+ prs, isEnd, err := downloader.GetPullRequests(1, 50)
+ assert.NoError(t, err)
+ assert.True(t, isEnd)
+ assert.Len(t, prs, 6)
+ prs, isEnd, err = downloader.GetPullRequests(1, 3)
+ assert.NoError(t, err)
+ assert.False(t, isEnd)
+ assert.Len(t, prs, 3)
+ assertPullRequestEqual(t, &base.PullRequest{
+ Number: 12,
+ PosterID: 689,
+ PosterName: "6543",
+ PosterEmail: "6543@obermui.de",
+ Title: "Dont Touch",
+ Content: "\r\nadd dont touch note",
+ Milestone: "V2 Finalize",
+ State: "closed",
+ IsLocked: false,
+ Created: time.Unix(1598982759, 0),
+ Updated: time.Unix(1599023425, 0),
+ Closed: timePtr(time.Unix(1598982934, 0)),
+ Assignees: []string{"techknowlogick"},
+ Base: base.PullRequestBranch{
+ CloneURL: "",
+ Ref: "master",
+ SHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
+ RepoName: "test_repo",
+ OwnerName: "gitea",
+ },
+ Head: base.PullRequestBranch{
+ CloneURL: "https://gitea.com/6543-forks/test_repo.git",
+ Ref: "refs/pull/12/head",
+ SHA: "b6ab5d9ae000b579a5fff03f92c486da4ddf48b6",
+ RepoName: "test_repo",
+ OwnerName: "6543-forks",
+ },
+ Merged: true,
+ MergedTime: timePtr(time.Unix(1598982934, 0)),
+ MergeCommitSHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
+ PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch",
+ }, prs[1])
+
+ reviews, err := downloader.GetReviews(base.BasicIssueContext(7))
+ assert.NoError(t, err)
+ assertReviewsEqual(t, []*base.Review{
+ {
+ ID: 1770,
+ IssueIndex: 7,
+ ReviewerID: 689,
+ ReviewerName: "6543",
+ CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
+ CreatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
+ State: "COMMENT", // TODO
+ Comments: []*base.ReviewComment{
+ {
+ ID: 116561,
+ InReplyTo: 0,
+ Content: "is one `\\newline` to less?",
+ TreePath: "README.md",
+ DiffHunk: "@@ -2,3 +2,3 @@\n \n-Test repository for testing migration from gitea 2 gitea\n\\ No newline at end of file\n+Test repository for testing migration from gitea 2 gitea",
+ Position: 0,
+ Line: 4,
+ CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
+ PosterID: 689,
+ Reactions: nil,
+ CreatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
+ UpdatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
+ },
+ },
+ },
+ {
+ ID: 1771,
+ IssueIndex: 7,
+ ReviewerID: 9,
+ ReviewerName: "techknowlogick",
+ CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
+ CreatedAt: time.Date(2020, 9, 1, 17, 6, 47, 0, time.UTC),
+ State: "REQUEST_CHANGES", // TODO
+ Content: "I think this needs some changes",
+ },
+ {
+ ID: 1772,
+ IssueIndex: 7,
+ ReviewerID: 9,
+ ReviewerName: "techknowlogick",
+ CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
+ CreatedAt: time.Date(2020, 9, 1, 17, 19, 51, 0, time.UTC),
+ State: base.ReviewStateApproved,
+ Official: true,
+ Content: "looks good",
+ },
+ }, reviews)
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/models/db"
+ "code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/log"
+ base "code.gitea.io/gitea/modules/migration"
+ repo_module "code.gitea.io/gitea/modules/repository"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/storage"
+ "code.gitea.io/gitea/modules/structs"
+ "code.gitea.io/gitea/modules/timeutil"
+ "code.gitea.io/gitea/modules/uri"
+ "code.gitea.io/gitea/services/pull"
+
+ gouuid "github.com/google/uuid"
+)
+
+var (
+ _ base.Uploader = &GiteaLocalUploader{}
+)
+
+// GiteaLocalUploader implements an Uploader to gitea sites
+type GiteaLocalUploader struct {
+ ctx context.Context
+ doer *models.User
+ repoOwner string
+ repoName string
+ repo *models.Repository
+ labels sync.Map
+ milestones sync.Map
+ issues sync.Map
+ gitRepo *git.Repository
+ prHeadCache map[string]struct{}
+ userMap map[int64]int64 // external user id mapping to user id
+ prCache map[int64]*models.PullRequest
+ gitServiceType structs.GitServiceType
+}
+
+// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
+func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
+ return &GiteaLocalUploader{
+ ctx: ctx,
+ doer: doer,
+ repoOwner: repoOwner,
+ repoName: repoName,
+ prHeadCache: make(map[string]struct{}),
+ userMap: make(map[int64]int64),
+ prCache: make(map[int64]*models.PullRequest),
+ }
+}
+
+// MaxBatchInsertSize returns the table's max batch insert size
+func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
+ switch tp {
+ case "issue":
+ return db.MaxBatchInsertSize(new(models.Issue))
+ case "comment":
+ return db.MaxBatchInsertSize(new(models.Comment))
+ case "milestone":
+ return db.MaxBatchInsertSize(new(models.Milestone))
+ case "label":
+ return db.MaxBatchInsertSize(new(models.Label))
+ case "release":
+ return db.MaxBatchInsertSize(new(models.Release))
+ case "pullrequest":
+ return db.MaxBatchInsertSize(new(models.PullRequest))
+ }
+ return 10
+}
+
+// CreateRepo creates a repository
+func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
+ owner, err := models.GetUserByName(g.repoOwner)
+ if err != nil {
+ return err
+ }
+
+ var r *models.Repository
+ if opts.MigrateToRepoID <= 0 {
+ r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{
+ Name: g.repoName,
+ Description: repo.Description,
+ OriginalURL: repo.OriginalURL,
+ GitServiceType: opts.GitServiceType,
+ IsPrivate: opts.Private,
+ IsMirror: opts.Mirror,
+ Status: models.RepositoryBeingMigrated,
+ })
+ } else {
+ r, err = models.GetRepositoryByID(opts.MigrateToRepoID)
+ }
+ if err != nil {
+ return err
+ }
+ r.DefaultBranch = repo.DefaultBranch
+
+ r, err = repo_module.MigrateRepositoryGitData(g.ctx, owner, r, base.MigrateOptions{
+ RepoName: g.repoName,
+ Description: repo.Description,
+ OriginalURL: repo.OriginalURL,
+ GitServiceType: opts.GitServiceType,
+ Mirror: repo.IsMirror,
+ LFS: opts.LFS,
+ LFSEndpoint: opts.LFSEndpoint,
+ CloneAddr: repo.CloneURL,
+ Private: repo.IsPrivate,
+ Wiki: opts.Wiki,
+ Releases: opts.Releases, // if didn't get releases, then sync them from tags
+ MirrorInterval: opts.MirrorInterval,
+ })
+
+ g.repo = r
+ if err != nil {
+ return err
+ }
+ g.gitRepo, err = git.OpenRepository(r.RepoPath())
+ return err
+}
+
+// Close closes this uploader
+func (g *GiteaLocalUploader) Close() {
+ if g.gitRepo != nil {
+ g.gitRepo.Close()
+ }
+}
+
+// CreateTopics creates topics
+func (g *GiteaLocalUploader) CreateTopics(topics ...string) error {
+ // ignore topics to long for the db
+ c := 0
+ for i := range topics {
+ if len(topics[i]) <= 50 {
+ topics[c] = topics[i]
+ c++
+ }
+ }
+ topics = topics[:c]
+ return models.SaveTopics(g.repo.ID, topics...)
+}
+
+// CreateMilestones creates milestones
+func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error {
+ var mss = make([]*models.Milestone, 0, len(milestones))
+ for _, milestone := range milestones {
+ var deadline timeutil.TimeStamp
+ if milestone.Deadline != nil {
+ deadline = timeutil.TimeStamp(milestone.Deadline.Unix())
+ }
+ if deadline == 0 {
+ deadline = timeutil.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.DefaultUILocation).Unix())
+ }
+
+ if milestone.Created.IsZero() {
+ if milestone.Updated != nil {
+ milestone.Created = *milestone.Updated
+ } else if milestone.Deadline != nil {
+ milestone.Created = *milestone.Deadline
+ } else {
+ milestone.Created = time.Now()
+ }
+ }
+ if milestone.Updated == nil || milestone.Updated.IsZero() {
+ milestone.Updated = &milestone.Created
+ }
+
+ var ms = models.Milestone{
+ RepoID: g.repo.ID,
+ Name: milestone.Title,
+ Content: milestone.Description,
+ IsClosed: milestone.State == "closed",
+ CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()),
+ UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()),
+ DeadlineUnix: deadline,
+ }
+ if ms.IsClosed && milestone.Closed != nil {
+ ms.ClosedDateUnix = timeutil.TimeStamp(milestone.Closed.Unix())
+ }
+ mss = append(mss, &ms)
+ }
+
+ err := models.InsertMilestones(mss...)
+ if err != nil {
+ return err
+ }
+
+ for _, ms := range mss {
+ g.milestones.Store(ms.Name, ms.ID)
+ }
+ return nil
+}
+
+// CreateLabels creates labels
+func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
+ var lbs = make([]*models.Label, 0, len(labels))
+ for _, label := range labels {
+ lbs = append(lbs, &models.Label{
+ RepoID: g.repo.ID,
+ Name: label.Name,
+ Description: label.Description,
+ Color: fmt.Sprintf("#%s", label.Color),
+ })
+ }
+
+ err := models.NewLabels(lbs...)
+ if err != nil {
+ return err
+ }
+ for _, lb := range lbs {
+ g.labels.Store(lb.Name, lb)
+ }
+ return nil
+}
+
+// CreateReleases creates releases
+func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
+ var rels = make([]*models.Release, 0, len(releases))
+ for _, release := range releases {
+ if release.Created.IsZero() {
+ if !release.Published.IsZero() {
+ release.Created = release.Published
+ } else {
+ release.Created = time.Now()
+ }
+ }
+
+ var rel = models.Release{
+ RepoID: g.repo.ID,
+ TagName: release.TagName,
+ LowerTagName: strings.ToLower(release.TagName),
+ Target: release.TargetCommitish,
+ Title: release.Name,
+ Sha1: release.TargetCommitish,
+ Note: release.Body,
+ IsDraft: release.Draft,
+ IsPrerelease: release.Prerelease,
+ IsTag: false,
+ CreatedUnix: timeutil.TimeStamp(release.Created.Unix()),
+ }
+
+ userid, ok := g.userMap[release.PublisherID]
+ tp := g.gitServiceType.Name()
+ if !ok && tp != "" {
+ var err error
+ userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID))
+ if err != nil {
+ log.Error("GetUserIDByExternalUserID: %v", err)
+ }
+ if userid > 0 {
+ g.userMap[release.PublisherID] = userid
+ }
+ }
+
+ if userid > 0 {
+ rel.PublisherID = userid
+ } else {
+ rel.PublisherID = g.doer.ID
+ rel.OriginalAuthor = release.PublisherName
+ rel.OriginalAuthorID = release.PublisherID
+ }
+
+ // calc NumCommits if no draft
+ if !release.Draft {
+ commit, err := g.gitRepo.GetTagCommit(rel.TagName)
+ if err != nil {
+ return fmt.Errorf("GetTagCommit[%v]: %v", rel.TagName, err)
+ }
+ rel.NumCommits, err = commit.CommitsCount()
+ if err != nil {
+ return fmt.Errorf("CommitsCount: %v", err)
+ }
+ }
+
+ for _, asset := range release.Assets {
+ if asset.Created.IsZero() {
+ if !asset.Updated.IsZero() {
+ asset.Created = asset.Updated
+ } else {
+ asset.Created = release.Created
+ }
+ }
+ var attach = models.Attachment{
+ UUID: gouuid.New().String(),
+ Name: asset.Name,
+ DownloadCount: int64(*asset.DownloadCount),
+ Size: int64(*asset.Size),
+ CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()),
+ }
+
+ // download attachment
+ err := func() error {
+ // asset.DownloadURL maybe a local file
+ var rc io.ReadCloser
+ var err error
+ if asset.DownloadFunc != nil {
+ rc, err = asset.DownloadFunc()
+ if err != nil {
+ return err
+ }
+ } else if asset.DownloadURL != nil {
+ rc, err = uri.Open(*asset.DownloadURL)
+ if err != nil {
+ return err
+ }
+ }
+ if rc == nil {
+ return nil
+ }
+ _, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size))
+ rc.Close()
+ return err
+ }()
+ if err != nil {
+ return err
+ }
+
+ rel.Attachments = append(rel.Attachments, &attach)
+ }
+
+ rels = append(rels, &rel)
+ }
+
+ return models.InsertReleases(rels...)
+}
+
+// SyncTags syncs releases with tags in the database
+func (g *GiteaLocalUploader) SyncTags() error {
+ return repo_module.SyncReleasesWithTags(g.repo, g.gitRepo)
+}
+
+// CreateIssues creates issues
+func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
+ var iss = make([]*models.Issue, 0, len(issues))
+ for _, issue := range issues {
+ var labels []*models.Label
+ for _, label := range issue.Labels {
+ lb, ok := g.labels.Load(label.Name)
+ if ok {
+ labels = append(labels, lb.(*models.Label))
+ }
+ }
+
+ var milestoneID int64
+ if issue.Milestone != "" {
+ milestone, ok := g.milestones.Load(issue.Milestone)
+ if ok {
+ milestoneID = milestone.(int64)
+ }
+ }
+
+ if issue.Created.IsZero() {
+ if issue.Closed != nil {
+ issue.Created = *issue.Closed
+ } else {
+ issue.Created = time.Now()
+ }
+ }
+ if issue.Updated.IsZero() {
+ if issue.Closed != nil {
+ issue.Updated = *issue.Closed
+ } else {
+ issue.Updated = time.Now()
+ }
+ }
+
+ var is = models.Issue{
+ RepoID: g.repo.ID,
+ Repo: g.repo,
+ Index: issue.Number,
+ Title: issue.Title,
+ Content: issue.Content,
+ Ref: issue.Ref,
+ IsClosed: issue.State == "closed",
+ IsLocked: issue.IsLocked,
+ MilestoneID: milestoneID,
+ Labels: labels,
+ CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()),
+ UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()),
+ }
+
+ userid, ok := g.userMap[issue.PosterID]
+ tp := g.gitServiceType.Name()
+ if !ok && tp != "" {
+ var err error
+ userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID))
+ if err != nil {
+ log.Error("GetUserIDByExternalUserID: %v", err)
+ }
+ if userid > 0 {
+ g.userMap[issue.PosterID] = userid
+ }
+ }
+
+ if userid > 0 {
+ is.PosterID = userid
+ } else {
+ is.PosterID = g.doer.ID
+ is.OriginalAuthor = issue.PosterName
+ is.OriginalAuthorID = issue.PosterID
+ }
+
+ if issue.Closed != nil {
+ is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix())
+ }
+ // add reactions
+ for _, reaction := range issue.Reactions {
+ userid, ok := g.userMap[reaction.UserID]
+ if !ok && tp != "" {
+ var err error
+ userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
+ if err != nil {
+ log.Error("GetUserIDByExternalUserID: %v", err)
+ }
+ if userid > 0 {
+ g.userMap[reaction.UserID] = userid
+ }
+ }
+ var res = models.Reaction{
+ Type: reaction.Content,
+ CreatedUnix: timeutil.TimeStampNow(),
+ }
+ if userid > 0 {
+ res.UserID = userid
+ } else {
+ res.UserID = g.doer.ID
+ res.OriginalAuthorID = reaction.UserID
+ res.OriginalAuthor = reaction.UserName
+ }
+ is.Reactions = append(is.Reactions, &res)
+ }
+ iss = append(iss, &is)
+ }
+
+ if len(iss) > 0 {
+ if err := models.InsertIssues(iss...); err != nil {
+ return err
+ }
+
+ for _, is := range iss {
+ g.issues.Store(is.Index, is)
+ }
+ }
+
+ return nil
+}
+
+// CreateComments creates comments of issues
+func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
+ var cms = make([]*models.Comment, 0, len(comments))
+ for _, comment := range comments {
+ var issue *models.Issue
+ issueInter, ok := g.issues.Load(comment.IssueIndex)
+ if !ok {
+ var err error
+ issue, err = models.GetIssueByIndex(g.repo.ID, comment.IssueIndex)
+ if err != nil {
+ return err
+ }
+ g.issues.Store(comment.IssueIndex, issue)
+ } else {
+ issue = issueInter.(*models.Issue)
+ }
+
+ userid, ok := g.userMap[comment.PosterID]
+ tp := g.gitServiceType.Name()
+ if !ok && tp != "" {
+ var err error
+ userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID))
+ if err != nil {
+ log.Error("GetUserIDByExternalUserID: %v", err)
+ }
+ if userid > 0 {
+ g.userMap[comment.PosterID] = userid
+ }
+ }
+
+ if comment.Created.IsZero() {
+ comment.Created = time.Unix(int64(issue.CreatedUnix), 0)
+ }
+ if comment.Updated.IsZero() {
+ comment.Updated = comment.Created
+ }
+
+ cm := models.Comment{
+ IssueID: issue.ID,
+ Type: models.CommentTypeComment,
+ Content: comment.Content,
+ CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()),
+ UpdatedUnix: timeutil.TimeStamp(comment.Updated.Unix()),
+ }
+
+ if userid > 0 {
+ cm.PosterID = userid
+ } else {
+ cm.PosterID = g.doer.ID
+ cm.OriginalAuthor = comment.PosterName
+ cm.OriginalAuthorID = comment.PosterID
+ }
+
+ // add reactions
+ for _, reaction := range comment.Reactions {
+ userid, ok := g.userMap[reaction.UserID]
+ if !ok && tp != "" {
+ var err error
+ userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
+ if err != nil {
+ log.Error("GetUserIDByExternalUserID: %v", err)
+ }
+ if userid > 0 {
+ g.userMap[reaction.UserID] = userid
+ }
+ }
+ var res = models.Reaction{
+ Type: reaction.Content,
+ CreatedUnix: timeutil.TimeStampNow(),
+ }
+ if userid > 0 {
+ res.UserID = userid
+ } else {
+ res.UserID = g.doer.ID
+ res.OriginalAuthorID = reaction.UserID
+ res.OriginalAuthor = reaction.UserName
+ }
+ cm.Reactions = append(cm.Reactions, &res)
+ }
+
+ cms = append(cms, &cm)
+ }
+
+ if len(cms) == 0 {
+ return nil
+ }
+ return models.InsertIssueComments(cms)
+}
+
+// CreatePullRequests creates pull requests
+func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error {
+ var gprs = make([]*models.PullRequest, 0, len(prs))
+ for _, pr := range prs {
+ gpr, err := g.newPullRequest(pr)
+ if err != nil {
+ return err
+ }
+
+ userid, ok := g.userMap[pr.PosterID]
+ tp := g.gitServiceType.Name()
+ if !ok && tp != "" {
+ var err error
+ userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
+ if err != nil {
+ log.Error("GetUserIDByExternalUserID: %v", err)
+ }
+ if userid > 0 {
+ g.userMap[pr.PosterID] = userid
+ }
+ }
+
+ if userid > 0 {
+ gpr.Issue.PosterID = userid
+ } else {
+ gpr.Issue.PosterID = g.doer.ID
+ gpr.Issue.OriginalAuthor = pr.PosterName
+ gpr.Issue.OriginalAuthorID = pr.PosterID
+ }
+
+ gprs = append(gprs, gpr)
+ }
+ if err := models.InsertPullRequests(gprs...); err != nil {
+ return err
+ }
+ for _, pr := range gprs {
+ g.issues.Store(pr.Issue.Index, pr.Issue)
+ pull.AddToTaskQueue(pr)
+ }
+ return nil
+}
+
+func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) {
+ var labels []*models.Label
+ for _, label := range pr.Labels {
+ lb, ok := g.labels.Load(label.Name)
+ if ok {
+ labels = append(labels, lb.(*models.Label))
+ }
+ }
+
+ var milestoneID int64
+ if pr.Milestone != "" {
+ milestone, ok := g.milestones.Load(pr.Milestone)
+ if ok {
+ milestoneID = milestone.(int64)
+ }
+ }
+
+ // download patch file
+ err := func() error {
+ if pr.PatchURL == "" {
+ return nil
+ }
+ // pr.PatchURL maybe a local file
+ ret, err := uri.Open(pr.PatchURL)
+ if err != nil {
+ return err
+ }
+ defer ret.Close()
+ pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
+ if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
+ return err
+ }
+ f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)))
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ _, err = io.Copy(f, ret)
+ return err
+ }()
+ if err != nil {
+ return nil, err
+ }
+
+ // set head information
+ pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
+ if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
+ return nil, err
+ }
+ p, err := os.Create(filepath.Join(pullHead, "head"))
+ if err != nil {
+ return nil, err
+ }
+ _, err = p.WriteString(pr.Head.SHA)
+ p.Close()
+ if err != nil {
+ return nil, err
+ }
+
+ var head = "unknown repository"
+ if pr.IsForkPullRequest() && pr.State != "closed" {
+ if pr.Head.OwnerName != "" {
+ remote := pr.Head.OwnerName
+ _, ok := g.prHeadCache[remote]
+ if !ok {
+ // git remote add
+ err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
+ if err != nil {
+ log.Error("AddRemote failed: %s", err)
+ } else {
+ g.prHeadCache[remote] = struct{}{}
+ ok = true
+ }
+ }
+
+ if ok {
+ _, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.repo.RepoPath())
+ if err != nil {
+ log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
+ } else {
+ headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
+ if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
+ return nil, err
+ }
+ b, err := os.Create(headBranch)
+ if err != nil {
+ return nil, err
+ }
+ _, err = b.WriteString(pr.Head.SHA)
+ b.Close()
+ if err != nil {
+ return nil, err
+ }
+ head = pr.Head.OwnerName + "/" + pr.Head.Ref
+ }
+ }
+ }
+ } else {
+ head = pr.Head.Ref
+ // Ensure the closed PR SHA still points to an existing ref
+ _, err = git.NewCommand("rev-list", "--quiet", "-1", pr.Head.SHA).RunInDir(g.repo.RepoPath())
+ if err != nil {
+ if pr.Head.SHA != "" {
+ // Git update-ref remove bad references with a relative path
+ log.Warn("Deprecated local head, removing : %v", pr.Head.SHA)
+ relPath := pr.GetGitRefName()
+ _, err = git.NewCommand("update-ref", "--no-deref", "-d", relPath).RunInDir(g.repo.RepoPath())
+ } else {
+ // The SHA is empty, remove the head file
+ log.Warn("Empty reference, removing : %v", pullHead)
+ err = os.Remove(filepath.Join(pullHead, "head"))
+ }
+ if err != nil {
+ log.Error("Cannot remove local head ref, %v", err)
+ }
+ }
+ }
+
+ if pr.Created.IsZero() {
+ if pr.Closed != nil {
+ pr.Created = *pr.Closed
+ } else if pr.MergedTime != nil {
+ pr.Created = *pr.MergedTime
+ } else {
+ pr.Created = time.Now()
+ }
+ }
+ if pr.Updated.IsZero() {
+ pr.Updated = pr.Created
+ }
+
+ var issue = models.Issue{
+ RepoID: g.repo.ID,
+ Repo: g.repo,
+ Title: pr.Title,
+ Index: pr.Number,
+ Content: pr.Content,
+ MilestoneID: milestoneID,
+ IsPull: true,
+ IsClosed: pr.State == "closed",
+ IsLocked: pr.IsLocked,
+ Labels: labels,
+ CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()),
+ UpdatedUnix: timeutil.TimeStamp(pr.Updated.Unix()),
+ }
+
+ tp := g.gitServiceType.Name()
+
+ userid, ok := g.userMap[pr.PosterID]
+ if !ok && tp != "" {
+ var err error
+ userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
+ if err != nil {
+ log.Error("GetUserIDByExternalUserID: %v", err)
+ }
+ if userid > 0 {
+ g.userMap[pr.PosterID] = userid
+ }
+ }
+
+ if userid > 0 {
+ issue.PosterID = userid
+ } else {
+ issue.PosterID = g.doer.ID
+ issue.OriginalAuthor = pr.PosterName
+ issue.OriginalAuthorID = pr.PosterID
+ }
+
+ // add reactions
+ for _, reaction := range pr.Reactions {
+ userid, ok := g.userMap[reaction.UserID]
+ if !ok && tp != "" {
+ var err error
+ userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
+ if err != nil {
+ log.Error("GetUserIDByExternalUserID: %v", err)
+ }
+ if userid > 0 {
+ g.userMap[reaction.UserID] = userid
+ }
+ }
+ var res = models.Reaction{
+ Type: reaction.Content,
+ CreatedUnix: timeutil.TimeStampNow(),
+ }
+ if userid > 0 {
+ res.UserID = userid
+ } else {
+ res.UserID = g.doer.ID
+ res.OriginalAuthorID = reaction.UserID
+ res.OriginalAuthor = reaction.UserName
+ }
+ issue.Reactions = append(issue.Reactions, &res)
+ }
+
+ var pullRequest = models.PullRequest{
+ HeadRepoID: g.repo.ID,
+ HeadBranch: head,
+ BaseRepoID: g.repo.ID,
+ BaseBranch: pr.Base.Ref,
+ MergeBase: pr.Base.SHA,
+ Index: pr.Number,
+ HasMerged: pr.Merged,
+
+ Issue: &issue,
+ }
+
+ if pullRequest.Issue.IsClosed && pr.Closed != nil {
+ pullRequest.Issue.ClosedUnix = timeutil.TimeStamp(pr.Closed.Unix())
+ }
+ if pullRequest.HasMerged && pr.MergedTime != nil {
+ pullRequest.MergedUnix = timeutil.TimeStamp(pr.MergedTime.Unix())
+ pullRequest.MergedCommitID = pr.MergeCommitSHA
+ pullRequest.MergerID = g.doer.ID
+ }
+
+ // TODO: assignees
+
+ return &pullRequest, nil
+}
+
+func convertReviewState(state string) models.ReviewType {
+ switch state {
+ case base.ReviewStatePending:
+ return models.ReviewTypePending
+ case base.ReviewStateApproved:
+ return models.ReviewTypeApprove
+ case base.ReviewStateChangesRequested:
+ return models.ReviewTypeReject
+ case base.ReviewStateCommented:
+ return models.ReviewTypeComment
+ default:
+ return models.ReviewTypePending
+ }
+}
+
+// CreateReviews create pull request reviews
+func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
+ var cms = make([]*models.Review, 0, len(reviews))
+ for _, review := range reviews {
+ var issue *models.Issue
+ issueInter, ok := g.issues.Load(review.IssueIndex)
+ if !ok {
+ var err error
+ issue, err = models.GetIssueByIndex(g.repo.ID, review.IssueIndex)
+ if err != nil {
+ return err
+ }
+ g.issues.Store(review.IssueIndex, issue)
+ } else {
+ issue = issueInter.(*models.Issue)
+ }
+
+ userid, ok := g.userMap[review.ReviewerID]
+ tp := g.gitServiceType.Name()
+ if !ok && tp != "" {
+ var err error
+ userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", review.ReviewerID))
+ if err != nil {
+ log.Error("GetUserIDByExternalUserID: %v", err)
+ }
+ if userid > 0 {
+ g.userMap[review.ReviewerID] = userid
+ }
+ }
+
+ if review.CreatedAt.IsZero() {
+ review.CreatedAt = time.Unix(int64(issue.CreatedUnix), 0)
+ }
+
+ var cm = models.Review{
+ Type: convertReviewState(review.State),
+ IssueID: issue.ID,
+ Content: review.Content,
+ Official: review.Official,
+ CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
+ UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
+ }
+
+ if userid > 0 {
+ cm.ReviewerID = userid
+ } else {
+ cm.ReviewerID = g.doer.ID
+ cm.OriginalAuthor = review.ReviewerName
+ cm.OriginalAuthorID = review.ReviewerID
+ }
+
+ // get pr
+ pr, ok := g.prCache[issue.ID]
+ if !ok {
+ var err error
+ pr, err = models.GetPullRequestByIssueIDWithNoAttributes(issue.ID)
+ if err != nil {
+ return err
+ }
+ g.prCache[issue.ID] = pr
+ }
+
+ for _, comment := range review.Comments {
+ line := comment.Line
+ if line != 0 {
+ comment.Position = 1
+ } else {
+ _, _, line, _ = git.ParseDiffHunkString(comment.DiffHunk)
+ }
+ headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName())
+ if err != nil {
+ log.Warn("GetRefCommitID[%s]: %v, the review comment will be ignored", pr.GetGitRefName(), err)
+ continue
+ }
+
+ var patch string
+ reader, writer := io.Pipe()
+ defer func() {
+ _ = reader.Close()
+ _ = writer.Close()
+ }()
+ go func() {
+ if err := git.GetRepoRawDiffForFile(g.gitRepo, pr.MergeBase, headCommitID, git.RawDiffNormal, comment.TreePath, writer); err != nil {
+ // We should ignore the error since the commit maybe removed when force push to the pull request
+ log.Warn("GetRepoRawDiffForFile failed when migrating [%s, %s, %s, %s]: %v", g.gitRepo.Path, pr.MergeBase, headCommitID, comment.TreePath, err)
+ }
+ _ = writer.Close()
+ }()
+
+ patch, _ = git.CutDiffAroundLine(reader, int64((&models.Comment{Line: int64(line + comment.Position - 1)}).UnsignedLine()), line < 0, setting.UI.CodeCommentLines)
+
+ if comment.CreatedAt.IsZero() {
+ comment.CreatedAt = review.CreatedAt
+ }
+ if comment.UpdatedAt.IsZero() {
+ comment.UpdatedAt = comment.CreatedAt
+ }
+
+ var c = models.Comment{
+ Type: models.CommentTypeCode,
+ PosterID: comment.PosterID,
+ IssueID: issue.ID,
+ Content: comment.Content,
+ Line: int64(line + comment.Position - 1),
+ TreePath: comment.TreePath,
+ CommitSHA: comment.CommitID,
+ Patch: patch,
+ CreatedUnix: timeutil.TimeStamp(comment.CreatedAt.Unix()),
+ UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()),
+ }
+
+ if userid > 0 {
+ c.PosterID = userid
+ } else {
+ c.PosterID = g.doer.ID
+ c.OriginalAuthor = review.ReviewerName
+ c.OriginalAuthorID = review.ReviewerID
+ }
+
+ cm.Comments = append(cm.Comments, &c)
+ }
+
+ cms = append(cms, &cm)
+ }
+
+ return models.InsertReviews(cms)
+}
+
+// Rollback when migrating failed, this will rollback all the changes.
+func (g *GiteaLocalUploader) Rollback() error {
+ if g.repo != nil && g.repo.ID > 0 {
+ g.gitRepo.Close()
+ if err := models.DeleteRepository(g.doer, g.repo.OwnerID, g.repo.ID); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// Finish when migrating success, this will do some status update things.
+func (g *GiteaLocalUploader) Finish() error {
+ if g.repo == nil || g.repo.ID <= 0 {
+ return ErrRepoNotCreated
+ }
+
+ // update issue_index
+ if err := models.RecalculateIssueIndexForRepo(g.repo.ID); err != nil {
+ return err
+ }
+
+ g.repo.Status = models.RepositoryReady
+ return models.UpdateRepositoryCols(g.repo, "status")
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/models/db"
+ "code.gitea.io/gitea/models/unittest"
+ "code.gitea.io/gitea/modules/graceful"
+ base "code.gitea.io/gitea/modules/migration"
+ "code.gitea.io/gitea/modules/structs"
+ "code.gitea.io/gitea/modules/util"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGiteaUploadRepo(t *testing.T) {
+ // FIXME: Since no accesskey or user/password will trigger rate limit of github, just skip
+ t.Skip()
+
+ unittest.PrepareTestEnv(t)
+
+ user := unittest.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
+
+ var (
+ downloader = NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", "", "go-xorm", "builder")
+ repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05")
+ uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
+ )
+
+ err := migrateRepository(downloader, uploader, base.MigrateOptions{
+ CloneAddr: "https://github.com/go-xorm/builder",
+ RepoName: repoName,
+ AuthUsername: "",
+
+ Wiki: true,
+ Issues: true,
+ Milestones: true,
+ Labels: true,
+ Releases: true,
+ Comments: true,
+ PullRequests: true,
+ Private: true,
+ Mirror: false,
+ }, nil)
+ assert.NoError(t, err)
+
+ repo := unittest.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)
+ assert.True(t, repo.HasWiki())
+ assert.EqualValues(t, models.RepositoryReady, repo.Status)
+
+ milestones, _, err := models.GetMilestones(models.GetMilestonesOption{
+ RepoID: repo.ID,
+ State: structs.StateOpen,
+ })
+ assert.NoError(t, err)
+ assert.Len(t, milestones, 1)
+
+ milestones, _, err = models.GetMilestones(models.GetMilestonesOption{
+ RepoID: repo.ID,
+ State: structs.StateClosed,
+ })
+ assert.NoError(t, err)
+ assert.Empty(t, milestones)
+
+ labels, err := models.GetLabelsByRepoID(repo.ID, "", db.ListOptions{})
+ assert.NoError(t, err)
+ assert.Len(t, labels, 12)
+
+ releases, err := models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
+ ListOptions: db.ListOptions{
+ PageSize: 10,
+ Page: 0,
+ },
+ IncludeTags: true,
+ })
+ assert.NoError(t, err)
+ assert.Len(t, releases, 8)
+
+ releases, err = models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
+ ListOptions: db.ListOptions{
+ PageSize: 10,
+ Page: 0,
+ },
+ IncludeTags: false,
+ })
+ assert.NoError(t, err)
+ assert.Len(t, releases, 1)
+
+ issues, err := models.Issues(&models.IssuesOptions{
+ RepoIDs: []int64{repo.ID},
+ IsPull: util.OptionalBoolFalse,
+ SortType: "oldest",
+ })
+ assert.NoError(t, err)
+ assert.Len(t, issues, 15)
+ assert.NoError(t, issues[0].LoadDiscussComments())
+ assert.Empty(t, issues[0].Comments)
+
+ pulls, _, err := models.PullRequests(repo.ID, &models.PullRequestsOptions{
+ SortType: "oldest",
+ })
+ assert.NoError(t, err)
+ assert.Len(t, pulls, 30)
+ assert.NoError(t, pulls[0].LoadIssue())
+ assert.NoError(t, pulls[0].Issue.LoadDiscussComments())
+ assert.Len(t, pulls[0].Issue.Comments, 2)
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "crypto/tls"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/modules/log"
+ base "code.gitea.io/gitea/modules/migration"
+ "code.gitea.io/gitea/modules/proxy"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/structs"
+ "code.gitea.io/gitea/modules/util"
+
+ "github.com/google/go-github/v39/github"
+ "golang.org/x/oauth2"
+)
+
+var (
+ _ base.Downloader = &GithubDownloaderV3{}
+ _ base.DownloaderFactory = &GithubDownloaderV3Factory{}
+ // GithubLimitRateRemaining limit to wait for new rate to apply
+ GithubLimitRateRemaining = 0
+)
+
+func init() {
+ RegisterDownloaderFactory(&GithubDownloaderV3Factory{})
+}
+
+// GithubDownloaderV3Factory defines a github downloader v3 factory
+type GithubDownloaderV3Factory struct {
+}
+
+// New returns a Downloader related to this factory according MigrateOptions
+func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
+ u, err := url.Parse(opts.CloneAddr)
+ if err != nil {
+ return nil, err
+ }
+
+ baseURL := u.Scheme + "://" + u.Host
+ fields := strings.Split(u.Path, "/")
+ oldOwner := fields[1]
+ oldName := strings.TrimSuffix(fields[2], ".git")
+
+ log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
+
+ return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
+}
+
+// GitServiceType returns the type of git service
+func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
+ return structs.GithubService
+}
+
+// GithubDownloaderV3 implements a Downloader interface to get repository information
+// from github via APIv3
+type GithubDownloaderV3 struct {
+ base.NullDownloader
+ ctx context.Context
+ clients []*github.Client
+ repoOwner string
+ repoName string
+ userName string
+ password string
+ rates []*github.Rate
+ curClientIdx int
+ maxPerPage int
+ SkipReactions bool
+}
+
+// NewGithubDownloaderV3 creates a github Downloader via github v3 API
+func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
+ var downloader = GithubDownloaderV3{
+ userName: userName,
+ password: password,
+ ctx: ctx,
+ repoOwner: repoOwner,
+ repoName: repoName,
+ maxPerPage: 100,
+ }
+
+ if token != "" {
+ tokens := strings.Split(token, ",")
+ for _, token := range tokens {
+ token = strings.TrimSpace(token)
+ ts := oauth2.StaticTokenSource(
+ &oauth2.Token{AccessToken: token},
+ )
+ var client = &http.Client{
+ Transport: &oauth2.Transport{
+ Base: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: func(req *http.Request) (*url.URL, error) {
+ return proxy.Proxy()(req)
+ },
+ },
+ Source: oauth2.ReuseTokenSource(nil, ts),
+ },
+ }
+
+ downloader.addClient(client, baseURL)
+ }
+ } else {
+ var client = &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: func(req *http.Request) (*url.URL, error) {
+ req.SetBasicAuth(userName, password)
+ return proxy.Proxy()(req)
+ },
+ },
+ }
+ downloader.addClient(client, baseURL)
+ }
+ return &downloader
+}
+
+func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {
+ githubClient := github.NewClient(client)
+ if baseURL != "https://github.com" {
+ githubClient, _ = github.NewEnterpriseClient(baseURL, baseURL, client)
+ }
+ g.clients = append(g.clients, githubClient)
+ g.rates = append(g.rates, nil)
+}
+
+// SetContext set context
+func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
+ g.ctx = ctx
+}
+
+func (g *GithubDownloaderV3) waitAndPickClient() {
+ var recentIdx int
+ var maxRemaining int
+ for i := 0; i < len(g.clients); i++ {
+ if g.rates[i] != nil && g.rates[i].Remaining > maxRemaining {
+ maxRemaining = g.rates[i].Remaining
+ recentIdx = i
+ }
+ }
+ g.curClientIdx = recentIdx // if no max remain, it will always pick the first client.
+
+ for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {
+ timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))
+ select {
+ case <-g.ctx.Done():
+ util.StopTimer(timer)
+ return
+ case <-timer.C:
+ }
+
+ err := g.RefreshRate()
+ if err != nil {
+ log.Error("g.getClient().RateLimits: %s", err)
+ }
+ }
+}
+
+// RefreshRate update the current rate (doesn't count in rate limit)
+func (g *GithubDownloaderV3) RefreshRate() error {
+ rates, _, err := g.getClient().RateLimits(g.ctx)
+ if err != nil {
+ // if rate limit is not enabled, ignore it
+ if strings.Contains(err.Error(), "404") {
+ g.setRate(nil)
+ return nil
+ }
+ return err
+ }
+
+ g.setRate(rates.GetCore())
+ return nil
+}
+
+func (g *GithubDownloaderV3) getClient() *github.Client {
+ return g.clients[g.curClientIdx]
+}
+
+func (g *GithubDownloaderV3) setRate(rate *github.Rate) {
+ g.rates[g.curClientIdx] = rate
+}
+
+// GetRepoInfo returns a repository information
+func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
+ g.waitAndPickClient()
+ gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+ if err != nil {
+ return nil, err
+ }
+ g.setRate(&resp.Rate)
+
+ // convert github repo to stand Repo
+ return &base.Repository{
+ Owner: g.repoOwner,
+ Name: gr.GetName(),
+ IsPrivate: gr.GetPrivate(),
+ Description: gr.GetDescription(),
+ OriginalURL: gr.GetHTMLURL(),
+ CloneURL: gr.GetCloneURL(),
+ DefaultBranch: gr.GetDefaultBranch(),
+ }, nil
+}
+
+// GetTopics return github topics
+func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
+ g.waitAndPickClient()
+ r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+ if err != nil {
+ return nil, err
+ }
+ g.setRate(&resp.Rate)
+ return r.Topics, nil
+}
+
+// GetMilestones returns milestones
+func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
+ var perPage = g.maxPerPage
+ var milestones = make([]*base.Milestone, 0, perPage)
+ for i := 1; ; i++ {
+ g.waitAndPickClient()
+ ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
+ &github.MilestoneListOptions{
+ State: "all",
+ ListOptions: github.ListOptions{
+ Page: i,
+ PerPage: perPage,
+ }})
+ if err != nil {
+ return nil, err
+ }
+ g.setRate(&resp.Rate)
+
+ for _, m := range ms {
+ var state = "open"
+ if m.State != nil {
+ state = *m.State
+ }
+ milestones = append(milestones, &base.Milestone{
+ Title: m.GetTitle(),
+ Description: m.GetDescription(),
+ Deadline: m.DueOn,
+ State: state,
+ Created: m.GetCreatedAt(),
+ Updated: m.UpdatedAt,
+ Closed: m.ClosedAt,
+ })
+ }
+ if len(ms) < perPage {
+ break
+ }
+ }
+ return milestones, nil
+}
+
+func convertGithubLabel(label *github.Label) *base.Label {
+ return &base.Label{
+ Name: label.GetName(),
+ Color: label.GetColor(),
+ Description: label.GetDescription(),
+ }
+}
+
+// GetLabels returns labels
+func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
+ var perPage = g.maxPerPage
+ var labels = make([]*base.Label, 0, perPage)
+ for i := 1; ; i++ {
+ g.waitAndPickClient()
+ ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
+ &github.ListOptions{
+ Page: i,
+ PerPage: perPage,
+ })
+ if err != nil {
+ return nil, err
+ }
+ g.setRate(&resp.Rate)
+
+ for _, label := range ls {
+ labels = append(labels, convertGithubLabel(label))
+ }
+ if len(ls) < perPage {
+ break
+ }
+ }
+ return labels, nil
+}
+
+func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release {
+ r := &base.Release{
+ Name: rel.GetName(),
+ TagName: rel.GetTagName(),
+ TargetCommitish: rel.GetTargetCommitish(),
+ Draft: rel.GetDraft(),
+ Prerelease: rel.GetPrerelease(),
+ Created: rel.GetCreatedAt().Time,
+ PublisherID: rel.GetAuthor().GetID(),
+ PublisherName: rel.GetAuthor().GetLogin(),
+ PublisherEmail: rel.GetAuthor().GetEmail(),
+ Body: rel.GetBody(),
+ }
+
+ if rel.PublishedAt != nil {
+ r.Published = rel.PublishedAt.Time
+ }
+
+ httpClient := &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }
+
+ for _, asset := range rel.Assets {
+ var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
+ r.Assets = append(r.Assets, &base.ReleaseAsset{
+ ID: asset.GetID(),
+ Name: asset.GetName(),
+ ContentType: asset.ContentType,
+ Size: asset.Size,
+ DownloadCount: asset.DownloadCount,
+ Created: asset.CreatedAt.Time,
+ Updated: asset.UpdatedAt.Time,
+ DownloadFunc: func() (io.ReadCloser, error) {
+ g.waitAndPickClient()
+ asset, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
+ if err != nil {
+ return nil, err
+ }
+ if err := g.RefreshRate(); err != nil {
+ log.Error("g.getClient().RateLimits: %s", err)
+ }
+ if asset == nil {
+ if redirectURL != "" {
+ g.waitAndPickClient()
+ req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := httpClient.Do(req)
+ err1 := g.RefreshRate()
+ if err1 != nil {
+ log.Error("g.getClient().RateLimits: %s", err1)
+ }
+ if err != nil {
+ return nil, err
+ }
+ return resp.Body, nil
+ }
+ return nil, fmt.Errorf("No release asset found for %d", assetID)
+ }
+ return asset, nil
+ },
+ })
+ }
+ return r
+}
+
+// GetReleases returns releases
+func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
+ var perPage = g.maxPerPage
+ var releases = make([]*base.Release, 0, perPage)
+ for i := 1; ; i++ {
+ g.waitAndPickClient()
+ ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
+ &github.ListOptions{
+ Page: i,
+ PerPage: perPage,
+ })
+ if err != nil {
+ return nil, err
+ }
+ g.setRate(&resp.Rate)
+
+ for _, release := range ls {
+ releases = append(releases, g.convertGithubRelease(release))
+ }
+ if len(ls) < perPage {
+ break
+ }
+ }
+ return releases, nil
+}
+
+// GetIssues returns issues according start and limit
+func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
+ if perPage > g.maxPerPage {
+ perPage = g.maxPerPage
+ }
+ opt := &github.IssueListByRepoOptions{
+ Sort: "created",
+ Direction: "asc",
+ State: "all",
+ ListOptions: github.ListOptions{
+ PerPage: perPage,
+ Page: page,
+ },
+ }
+
+ var allIssues = make([]*base.Issue, 0, perPage)
+ g.waitAndPickClient()
+ issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing repos: %v", err)
+ }
+ log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
+ g.setRate(&resp.Rate)
+ for _, issue := range issues {
+ if issue.IsPullRequest() {
+ continue
+ }
+
+ var labels = make([]*base.Label, 0, len(issue.Labels))
+ for _, l := range issue.Labels {
+ labels = append(labels, convertGithubLabel(l))
+ }
+
+ // get reactions
+ var reactions []*base.Reaction
+ if !g.SkipReactions {
+ for i := 1; ; i++ {
+ g.waitAndPickClient()
+ res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
+ Page: i,
+ PerPage: perPage,
+ })
+ if err != nil {
+ return nil, false, err
+ }
+ g.setRate(&resp.Rate)
+ if len(res) == 0 {
+ break
+ }
+ for _, reaction := range res {
+ reactions = append(reactions, &base.Reaction{
+ UserID: reaction.User.GetID(),
+ UserName: reaction.User.GetLogin(),
+ Content: reaction.GetContent(),
+ })
+ }
+ }
+ }
+
+ var assignees []string
+ for i := range issue.Assignees {
+ assignees = append(assignees, issue.Assignees[i].GetLogin())
+ }
+
+ allIssues = append(allIssues, &base.Issue{
+ Title: *issue.Title,
+ Number: int64(*issue.Number),
+ PosterID: issue.GetUser().GetID(),
+ PosterName: issue.GetUser().GetLogin(),
+ PosterEmail: issue.GetUser().GetEmail(),
+ Content: issue.GetBody(),
+ Milestone: issue.GetMilestone().GetTitle(),
+ State: issue.GetState(),
+ Created: issue.GetCreatedAt(),
+ Updated: issue.GetUpdatedAt(),
+ Labels: labels,
+ Reactions: reactions,
+ Closed: issue.ClosedAt,
+ IsLocked: issue.GetLocked(),
+ Assignees: assignees,
+ Context: base.BasicIssueContext(*issue.Number),
+ })
+ }
+
+ return allIssues, len(issues) < perPage, nil
+}
+
+// SupportGetRepoComments return true if it supports get repo comments
+func (g *GithubDownloaderV3) SupportGetRepoComments() bool {
+ return true
+}
+
+// GetComments returns comments according issueNumber
+func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
+ if opts.Context != nil {
+ comments, err := g.getComments(opts.Context)
+ return comments, false, err
+ }
+
+ return g.GetAllComments(opts.Page, opts.PageSize)
+}
+
+func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*base.Comment, error) {
+ var (
+ allComments = make([]*base.Comment, 0, g.maxPerPage)
+ created = "created"
+ asc = "asc"
+ )
+ opt := &github.IssueListCommentsOptions{
+ Sort: &created,
+ Direction: &asc,
+ ListOptions: github.ListOptions{
+ PerPage: g.maxPerPage,
+ },
+ }
+ for {
+ g.waitAndPickClient()
+ comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt)
+ if err != nil {
+ return nil, fmt.Errorf("error while listing repos: %v", err)
+ }
+ g.setRate(&resp.Rate)
+ for _, comment := range comments {
+ // get reactions
+ var reactions []*base.Reaction
+ if !g.SkipReactions {
+ for i := 1; ; i++ {
+ g.waitAndPickClient()
+ res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
+ Page: i,
+ PerPage: g.maxPerPage,
+ })
+ if err != nil {
+ return nil, err
+ }
+ g.setRate(&resp.Rate)
+ if len(res) == 0 {
+ break
+ }
+ for _, reaction := range res {
+ reactions = append(reactions, &base.Reaction{
+ UserID: reaction.User.GetID(),
+ UserName: reaction.User.GetLogin(),
+ Content: reaction.GetContent(),
+ })
+ }
+ }
+ }
+
+ allComments = append(allComments, &base.Comment{
+ IssueIndex: issueContext.LocalID(),
+ PosterID: comment.GetUser().GetID(),
+ PosterName: comment.GetUser().GetLogin(),
+ PosterEmail: comment.GetUser().GetEmail(),
+ Content: comment.GetBody(),
+ Created: comment.GetCreatedAt(),
+ Updated: comment.GetUpdatedAt(),
+ Reactions: reactions,
+ })
+ }
+ if resp.NextPage == 0 {
+ break
+ }
+ opt.Page = resp.NextPage
+ }
+ return allComments, nil
+}
+
+// GetAllComments returns repository comments according page and perPageSize
+func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) {
+ var (
+ allComments = make([]*base.Comment, 0, perPage)
+ created = "created"
+ asc = "asc"
+ )
+ if perPage > g.maxPerPage {
+ perPage = g.maxPerPage
+ }
+ opt := &github.IssueListCommentsOptions{
+ Sort: &created,
+ Direction: &asc,
+ ListOptions: github.ListOptions{
+ Page: page,
+ PerPage: perPage,
+ },
+ }
+
+ g.waitAndPickClient()
+ comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt)
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing repos: %v", err)
+ }
+ var isEnd = resp.NextPage == 0
+
+ log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage)
+ g.setRate(&resp.Rate)
+ for _, comment := range comments {
+ // get reactions
+ var reactions []*base.Reaction
+ if !g.SkipReactions {
+ for i := 1; ; i++ {
+ g.waitAndPickClient()
+ res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
+ Page: i,
+ PerPage: g.maxPerPage,
+ })
+ if err != nil {
+ return nil, false, err
+ }
+ g.setRate(&resp.Rate)
+ if len(res) == 0 {
+ break
+ }
+ for _, reaction := range res {
+ reactions = append(reactions, &base.Reaction{
+ UserID: reaction.User.GetID(),
+ UserName: reaction.User.GetLogin(),
+ Content: reaction.GetContent(),
+ })
+ }
+ }
+ }
+ idx := strings.LastIndex(*comment.IssueURL, "/")
+ issueIndex, _ := strconv.ParseInt((*comment.IssueURL)[idx+1:], 10, 64)
+ allComments = append(allComments, &base.Comment{
+ IssueIndex: issueIndex,
+ PosterID: comment.GetUser().GetID(),
+ PosterName: comment.GetUser().GetLogin(),
+ PosterEmail: comment.GetUser().GetEmail(),
+ Content: comment.GetBody(),
+ Created: comment.GetCreatedAt(),
+ Updated: comment.GetUpdatedAt(),
+ Reactions: reactions,
+ })
+ }
+
+ return allComments, isEnd, nil
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
+ if perPage > g.maxPerPage {
+ perPage = g.maxPerPage
+ }
+ opt := &github.PullRequestListOptions{
+ Sort: "created",
+ Direction: "asc",
+ State: "all",
+ ListOptions: github.ListOptions{
+ PerPage: perPage,
+ Page: page,
+ },
+ }
+ var allPRs = make([]*base.PullRequest, 0, perPage)
+ g.waitAndPickClient()
+ prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing repos: %v", err)
+ }
+ log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
+ g.setRate(&resp.Rate)
+ for _, pr := range prs {
+ var labels = make([]*base.Label, 0, len(pr.Labels))
+ for _, l := range pr.Labels {
+ labels = append(labels, convertGithubLabel(l))
+ }
+
+ // get reactions
+ var reactions []*base.Reaction
+ if !g.SkipReactions {
+ for i := 1; ; i++ {
+ g.waitAndPickClient()
+ res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
+ Page: i,
+ PerPage: perPage,
+ })
+ if err != nil {
+ return nil, false, err
+ }
+ g.setRate(&resp.Rate)
+ if len(res) == 0 {
+ break
+ }
+ for _, reaction := range res {
+ reactions = append(reactions, &base.Reaction{
+ UserID: reaction.User.GetID(),
+ UserName: reaction.User.GetLogin(),
+ Content: reaction.GetContent(),
+ })
+ }
+ }
+ }
+
+ // download patch and saved as tmp file
+ g.waitAndPickClient()
+
+ allPRs = append(allPRs, &base.PullRequest{
+ Title: pr.GetTitle(),
+ Number: int64(pr.GetNumber()),
+ PosterID: pr.GetUser().GetID(),
+ PosterName: pr.GetUser().GetLogin(),
+ PosterEmail: pr.GetUser().GetEmail(),
+ Content: pr.GetBody(),
+ Milestone: pr.GetMilestone().GetTitle(),
+ State: pr.GetState(),
+ Created: pr.GetCreatedAt(),
+ Updated: pr.GetUpdatedAt(),
+ Closed: pr.ClosedAt,
+ Labels: labels,
+ Merged: pr.MergedAt != nil,
+ MergeCommitSHA: pr.GetMergeCommitSHA(),
+ MergedTime: pr.MergedAt,
+ IsLocked: pr.ActiveLockReason != nil,
+ Head: base.PullRequestBranch{
+ Ref: pr.GetHead().GetRef(),
+ SHA: pr.GetHead().GetSHA(),
+ OwnerName: pr.GetHead().GetUser().GetLogin(),
+ RepoName: pr.GetHead().GetRepo().GetName(),
+ CloneURL: pr.GetHead().GetRepo().GetCloneURL(),
+ },
+ Base: base.PullRequestBranch{
+ Ref: pr.GetBase().GetRef(),
+ SHA: pr.GetBase().GetSHA(),
+ RepoName: pr.GetBase().GetRepo().GetName(),
+ OwnerName: pr.GetBase().GetUser().GetLogin(),
+ },
+ PatchURL: pr.GetPatchURL(),
+ Reactions: reactions,
+ Context: base.BasicIssueContext(*pr.Number),
+ })
+ }
+
+ return allPRs, len(prs) < perPage, nil
+}
+
+func convertGithubReview(r *github.PullRequestReview) *base.Review {
+ return &base.Review{
+ ID: r.GetID(),
+ ReviewerID: r.GetUser().GetID(),
+ ReviewerName: r.GetUser().GetLogin(),
+ CommitID: r.GetCommitID(),
+ Content: r.GetBody(),
+ CreatedAt: r.GetSubmittedAt(),
+ State: r.GetState(),
+ }
+}
+
+func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
+ var rcs = make([]*base.ReviewComment, 0, len(cs))
+ for _, c := range cs {
+ // get reactions
+ var reactions []*base.Reaction
+ if !g.SkipReactions {
+ for i := 1; ; i++ {
+ g.waitAndPickClient()
+ res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
+ Page: i,
+ PerPage: g.maxPerPage,
+ })
+ if err != nil {
+ return nil, err
+ }
+ g.setRate(&resp.Rate)
+ if len(res) == 0 {
+ break
+ }
+ for _, reaction := range res {
+ reactions = append(reactions, &base.Reaction{
+ UserID: reaction.User.GetID(),
+ UserName: reaction.User.GetLogin(),
+ Content: reaction.GetContent(),
+ })
+ }
+ }
+ }
+
+ rcs = append(rcs, &base.ReviewComment{
+ ID: c.GetID(),
+ InReplyTo: c.GetInReplyTo(),
+ Content: c.GetBody(),
+ TreePath: c.GetPath(),
+ DiffHunk: c.GetDiffHunk(),
+ Position: c.GetPosition(),
+ CommitID: c.GetCommitID(),
+ PosterID: c.GetUser().GetID(),
+ Reactions: reactions,
+ CreatedAt: c.GetCreatedAt(),
+ UpdatedAt: c.GetUpdatedAt(),
+ })
+ }
+ return rcs, nil
+}
+
+// GetReviews returns pull requests review
+func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Review, error) {
+ var allReviews = make([]*base.Review, 0, g.maxPerPage)
+ opt := &github.ListOptions{
+ PerPage: g.maxPerPage,
+ }
+ for {
+ g.waitAndPickClient()
+ reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt)
+ if err != nil {
+ return nil, fmt.Errorf("error while listing repos: %v", err)
+ }
+ g.setRate(&resp.Rate)
+ for _, review := range reviews {
+ r := convertGithubReview(review)
+ r.IssueIndex = context.LocalID()
+ // retrieve all review comments
+ opt2 := &github.ListOptions{
+ PerPage: g.maxPerPage,
+ }
+ for {
+ g.waitAndPickClient()
+ reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2)
+ if err != nil {
+ return nil, fmt.Errorf("error while listing repos: %v", err)
+ }
+ g.setRate(&resp.Rate)
+
+ cs, err := g.convertGithubReviewComments(reviewComments)
+ if err != nil {
+ return nil, err
+ }
+ r.Comments = append(r.Comments, cs...)
+ if resp.NextPage == 0 {
+ break
+ }
+ opt2.Page = resp.NextPage
+ }
+ allReviews = append(allReviews, r)
+ }
+ if resp.NextPage == 0 {
+ break
+ }
+ opt.Page = resp.NextPage
+ }
+ return allReviews, nil
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "os"
+ "testing"
+ "time"
+
+ base "code.gitea.io/gitea/modules/migration"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGitHubDownloadRepo(t *testing.T) {
+ GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in //
+ downloader := NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", os.Getenv("GITHUB_READ_TOKEN"), "go-gitea", "test_repo")
+ err := downloader.RefreshRate()
+ assert.NoError(t, err)
+
+ repo, err := downloader.GetRepoInfo()
+ assert.NoError(t, err)
+ assertRepositoryEqual(t, &base.Repository{
+ Name: "test_repo",
+ Owner: "go-gitea",
+ Description: "Test repository for testing migration from github to gitea",
+ CloneURL: "https://github.com/go-gitea/test_repo.git",
+ OriginalURL: "https://github.com/go-gitea/test_repo",
+ DefaultBranch: "master",
+ }, repo)
+
+ topics, err := downloader.GetTopics()
+ assert.NoError(t, err)
+ assert.Contains(t, topics, "gitea")
+
+ milestones, err := downloader.GetMilestones()
+ assert.NoError(t, err)
+ assertMilestonesEqual(t, []*base.Milestone{
+ {
+ Title: "1.0.0",
+ Description: "Milestone 1.0.0",
+ Deadline: timePtr(time.Date(2019, 11, 11, 8, 0, 0, 0, time.UTC)),
+ Created: time.Date(2019, 11, 12, 19, 37, 8, 0, time.UTC),
+ Updated: timePtr(time.Date(2019, 11, 12, 21, 56, 17, 0, time.UTC)),
+ Closed: timePtr(time.Date(2019, 11, 12, 19, 45, 49, 0, time.UTC)),
+ State: "closed",
+ },
+ {
+ Title: "1.1.0",
+ Description: "Milestone 1.1.0",
+ Deadline: timePtr(time.Date(2019, 11, 12, 8, 0, 0, 0, time.UTC)),
+ Created: time.Date(2019, 11, 12, 19, 37, 25, 0, time.UTC),
+ Updated: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
+ Closed: timePtr(time.Date(2019, 11, 12, 19, 45, 46, 0, time.UTC)),
+ State: "closed",
+ },
+ }, milestones)
+
+ labels, err := downloader.GetLabels()
+ assert.NoError(t, err)
+ assertLabelsEqual(t, []*base.Label{
+ {
+ Name: "bug",
+ Color: "d73a4a",
+ Description: "Something isn't working",
+ },
+ {
+ Name: "documentation",
+ Color: "0075ca",
+ Description: "Improvements or additions to documentation",
+ },
+ {
+ Name: "duplicate",
+ Color: "cfd3d7",
+ Description: "This issue or pull request already exists",
+ },
+ {
+ Name: "enhancement",
+ Color: "a2eeef",
+ Description: "New feature or request",
+ },
+ {
+ Name: "good first issue",
+ Color: "7057ff",
+ Description: "Good for newcomers",
+ },
+ {
+ Name: "help wanted",
+ Color: "008672",
+ Description: "Extra attention is needed",
+ },
+ {
+ Name: "invalid",
+ Color: "e4e669",
+ Description: "This doesn't seem right",
+ },
+ {
+ Name: "question",
+ Color: "d876e3",
+ Description: "Further information is requested",
+ },
+ {
+ Name: "wontfix",
+ Color: "ffffff",
+ Description: "This will not be worked on",
+ },
+ }, labels)
+
+ releases, err := downloader.GetReleases()
+ assert.NoError(t, err)
+ assertReleasesEqual(t, []*base.Release{
+ {
+ TagName: "v0.9.99",
+ TargetCommitish: "master",
+ Name: "First Release",
+ Body: "A test release",
+ Created: time.Date(2019, 11, 9, 16, 49, 21, 0, time.UTC),
+ Published: time.Date(2019, 11, 12, 20, 12, 10, 0, time.UTC),
+ PublisherID: 1669571,
+ PublisherName: "mrsdizzie",
+ },
+ }, releases)
+
+ // downloader.GetIssues()
+ issues, isEnd, err := downloader.GetIssues(1, 2)
+ assert.NoError(t, err)
+ assert.False(t, isEnd)
+ assertIssuesEqual(t, []*base.Issue{
+ {
+ Number: 1,
+ Title: "Please add an animated gif icon to the merge button",
+ Content: "I just want the merge button to hurt my eyes a little. \xF0\x9F\x98\x9D ",
+ Milestone: "1.0.0",
+ PosterID: 18600385,
+ PosterName: "guillep2k",
+ State: "closed",
+ Created: time.Date(2019, 11, 9, 17, 0, 29, 0, time.UTC),
+ Updated: time.Date(2019, 11, 12, 20, 29, 53, 0, time.UTC),
+ Labels: []*base.Label{
+ {
+ Name: "bug",
+ Color: "d73a4a",
+ Description: "Something isn't working",
+ },
+ {
+ Name: "good first issue",
+ Color: "7057ff",
+ Description: "Good for newcomers",
+ },
+ },
+ Reactions: []*base.Reaction{
+ {
+ UserID: 1669571,
+ UserName: "mrsdizzie",
+ Content: "+1",
+ },
+ },
+ Closed: timePtr(time.Date(2019, 11, 12, 20, 22, 22, 0, time.UTC)),
+ },
+ {
+ Number: 2,
+ Title: "Test issue",
+ Content: "This is test issue 2, do not touch!",
+ Milestone: "1.1.0",
+ PosterID: 1669571,
+ PosterName: "mrsdizzie",
+ State: "closed",
+ Created: time.Date(2019, 11, 12, 21, 0, 6, 0, time.UTC),
+ Updated: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
+ Labels: []*base.Label{
+ {
+ Name: "duplicate",
+ Color: "cfd3d7",
+ Description: "This issue or pull request already exists",
+ },
+ },
+ Reactions: []*base.Reaction{
+ {
+ UserID: 1669571,
+ UserName: "mrsdizzie",
+ Content: "heart",
+ },
+ {
+ UserID: 1669571,
+ UserName: "mrsdizzie",
+ Content: "laugh",
+ },
+ {
+ UserID: 1669571,
+ UserName: "mrsdizzie",
+ Content: "-1",
+ },
+ {
+ UserID: 1669571,
+ UserName: "mrsdizzie",
+ Content: "confused",
+ },
+ {
+ UserID: 1669571,
+ UserName: "mrsdizzie",
+ Content: "hooray",
+ },
+ {
+ UserID: 1669571,
+ UserName: "mrsdizzie",
+ Content: "+1",
+ },
+ },
+ Closed: timePtr(time.Date(2019, 11, 12, 21, 1, 31, 0, time.UTC)),
+ },
+ }, issues)
+
+ // downloader.GetComments()
+ comments, _, err := downloader.GetComments(base.GetCommentOptions{
+ Context: base.BasicIssueContext(2),
+ })
+ assert.NoError(t, err)
+ assertCommentsEqual(t, []*base.Comment{
+ {
+ IssueIndex: 2,
+ PosterID: 1669571,
+ PosterName: "mrsdizzie",
+ Created: time.Date(2019, 11, 12, 21, 0, 13, 0, time.UTC),
+ Updated: time.Date(2019, 11, 12, 21, 0, 13, 0, time.UTC),
+ Content: "This is a comment",
+ Reactions: []*base.Reaction{
+ {
+ UserID: 1669571,
+ UserName: "mrsdizzie",
+ Content: "+1",
+ },
+ },
+ },
+ {
+ IssueIndex: 2,
+ PosterID: 1669571,
+ PosterName: "mrsdizzie",
+ Created: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
+ Updated: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
+ Content: "A second comment",
+ Reactions: nil,
+ },
+ }, comments)
+
+ // downloader.GetPullRequests()
+ prs, _, err := downloader.GetPullRequests(1, 2)
+ assert.NoError(t, err)
+ assertPullRequestsEqual(t, []*base.PullRequest{
+ {
+ Number: 3,
+ Title: "Update README.md",
+ Content: "add warning to readme",
+ Milestone: "1.1.0",
+ PosterID: 1669571,
+ PosterName: "mrsdizzie",
+ State: "closed",
+ Created: time.Date(2019, 11, 12, 21, 21, 43, 0, time.UTC),
+ Updated: time.Date(2019, 11, 12, 21, 39, 28, 0, time.UTC),
+ Labels: []*base.Label{
+ {
+ Name: "documentation",
+ Color: "0075ca",
+ Description: "Improvements or additions to documentation",
+ },
+ },
+ PatchURL: "https://github.com/go-gitea/test_repo/pull/3.patch",
+ Head: base.PullRequestBranch{
+ Ref: "master",
+ CloneURL: "https://github.com/mrsdizzie/test_repo.git",
+ SHA: "076160cf0b039f13e5eff19619932d181269414b",
+ RepoName: "test_repo",
+
+ OwnerName: "mrsdizzie",
+ },
+ Base: base.PullRequestBranch{
+ Ref: "master",
+ SHA: "72866af952e98d02a73003501836074b286a78f6",
+ OwnerName: "go-gitea",
+ RepoName: "test_repo",
+ },
+ Closed: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
+ Merged: true,
+ MergedTime: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
+ MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
+ Context: base.BasicIssueContext(3),
+ },
+ {
+ Number: 4,
+ Title: "Test branch",
+ Content: "do not merge this PR",
+ Milestone: "1.0.0",
+ PosterID: 1669571,
+ PosterName: "mrsdizzie",
+ State: "open",
+ Created: time.Date(2019, 11, 12, 21, 54, 18, 0, time.UTC),
+ Updated: time.Date(2020, 1, 4, 11, 30, 1, 0, time.UTC),
+ Labels: []*base.Label{
+ {
+ Name: "bug",
+ Color: "d73a4a",
+ Description: "Something isn't working",
+ },
+ },
+ PatchURL: "https://github.com/go-gitea/test_repo/pull/4.patch",
+ Head: base.PullRequestBranch{
+ Ref: "test-branch",
+ SHA: "2be9101c543658591222acbee3eb799edfc3853d",
+ RepoName: "test_repo",
+ OwnerName: "mrsdizzie",
+ CloneURL: "https://github.com/mrsdizzie/test_repo.git",
+ },
+ Base: base.PullRequestBranch{
+ Ref: "master",
+ SHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
+ OwnerName: "go-gitea",
+ RepoName: "test_repo",
+ },
+ Merged: false,
+ MergeCommitSHA: "565d1208f5fffdc1c5ae1a2436491eb9a5e4ebae",
+ Reactions: []*base.Reaction{
+ {
+ UserID: 81045,
+ UserName: "lunny",
+ Content: "heart",
+ },
+ {
+ UserID: 81045,
+ UserName: "lunny",
+ Content: "+1",
+ },
+ },
+ Context: base.BasicIssueContext(4),
+ },
+ }, prs)
+
+ reviews, err := downloader.GetReviews(base.BasicIssueContext(3))
+ assert.NoError(t, err)
+ assertReviewsEqual(t, []*base.Review{
+ {
+ ID: 315859956,
+ IssueIndex: 3,
+ ReviewerID: 42128690,
+ ReviewerName: "jolheiser",
+ CommitID: "076160cf0b039f13e5eff19619932d181269414b",
+ CreatedAt: time.Date(2019, 11, 12, 21, 35, 24, 0, time.UTC),
+ State: base.ReviewStateApproved,
+ },
+ {
+ ID: 315860062,
+ IssueIndex: 3,
+ ReviewerID: 1824502,
+ ReviewerName: "zeripath",
+ CommitID: "076160cf0b039f13e5eff19619932d181269414b",
+ CreatedAt: time.Date(2019, 11, 12, 21, 35, 36, 0, time.UTC),
+ State: base.ReviewStateApproved,
+ },
+ {
+ ID: 315861440,
+ IssueIndex: 3,
+ ReviewerID: 165205,
+ ReviewerName: "lafriks",
+ CommitID: "076160cf0b039f13e5eff19619932d181269414b",
+ CreatedAt: time.Date(2019, 11, 12, 21, 38, 00, 0, time.UTC),
+ State: base.ReviewStateApproved,
+ },
+ }, reviews)
+
+ reviews, err = downloader.GetReviews(base.BasicIssueContext(4))
+ assert.NoError(t, err)
+ assertReviewsEqual(t, []*base.Review{
+ {
+ ID: 338338740,
+ IssueIndex: 4,
+ ReviewerID: 81045,
+ ReviewerName: "lunny",
+ CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
+ CreatedAt: time.Date(2020, 01, 04, 05, 33, 18, 0, time.UTC),
+ State: base.ReviewStateApproved,
+ Comments: []*base.ReviewComment{
+ {
+ ID: 363017488,
+ Content: "This is a good pull request.",
+ TreePath: "README.md",
+ DiffHunk: "@@ -1,2 +1,4 @@\n # test_repo\n Test repository for testing migration from github to gitea\n+",
+ Position: 3,
+ CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
+ PosterID: 81045,
+ CreatedAt: time.Date(2020, 01, 04, 05, 33, 06, 0, time.UTC),
+ UpdatedAt: time.Date(2020, 01, 04, 05, 33, 18, 0, time.UTC),
+ },
+ },
+ },
+ {
+ ID: 338339651,
+ IssueIndex: 4,
+ ReviewerID: 81045,
+ ReviewerName: "lunny",
+ CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
+ CreatedAt: time.Date(2020, 01, 04, 06, 07, 06, 0, time.UTC),
+ State: base.ReviewStateChangesRequested,
+ Content: "Don't add more reviews",
+ },
+ {
+ ID: 338349019,
+ IssueIndex: 4,
+ ReviewerID: 81045,
+ ReviewerName: "lunny",
+ CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
+ CreatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
+ State: base.ReviewStateCommented,
+ Comments: []*base.ReviewComment{
+ {
+ ID: 363029944,
+ Content: "test a single comment.",
+ TreePath: "LICENSE",
+ DiffHunk: "@@ -19,3 +19,5 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n SOFTWARE.\n+",
+ Position: 4,
+ CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
+ PosterID: 81045,
+ CreatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
+ UpdatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
+ },
+ },
+ },
+ }, reviews)
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "path"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/modules/log"
+ base "code.gitea.io/gitea/modules/migration"
+ "code.gitea.io/gitea/modules/proxy"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/structs"
+
+ "github.com/xanzy/go-gitlab"
+)
+
+var (
+ _ base.Downloader = &GitlabDownloader{}
+ _ base.DownloaderFactory = &GitlabDownloaderFactory{}
+)
+
+func init() {
+ RegisterDownloaderFactory(&GitlabDownloaderFactory{})
+}
+
+// GitlabDownloaderFactory defines a gitlab downloader factory
+type GitlabDownloaderFactory struct {
+}
+
+// New returns a Downloader related to this factory according MigrateOptions
+func (f *GitlabDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
+ u, err := url.Parse(opts.CloneAddr)
+ if err != nil {
+ return nil, err
+ }
+
+ baseURL := u.Scheme + "://" + u.Host
+ repoNameSpace := strings.TrimPrefix(u.Path, "/")
+ repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
+
+ log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
+
+ return NewGitlabDownloader(ctx, baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken)
+}
+
+// GitServiceType returns the type of git service
+func (f *GitlabDownloaderFactory) GitServiceType() structs.GitServiceType {
+ return structs.GitlabService
+}
+
+// GitlabDownloader implements a Downloader interface to get repository information
+// from gitlab via go-gitlab
+// - issueCount is incremented in GetIssues() to ensure PR and Issue numbers do not overlap,
+// because Gitlab has individual Issue and Pull Request numbers.
+type GitlabDownloader struct {
+ base.NullDownloader
+ ctx context.Context
+ client *gitlab.Client
+ repoID int
+ repoName string
+ issueCount int64
+ maxPerPage int
+}
+
+// NewGitlabDownloader creates a gitlab Downloader via gitlab API
+// Use either a username/password, personal token entered into the username field, or anonymous/public access
+// Note: Public access only allows very basic access
+func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
+ gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }))
+ // Only use basic auth if token is blank and password is NOT
+ // Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
+ if token == "" && password != "" {
+ gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
+ }
+
+ if err != nil {
+ log.Trace("Error logging into gitlab: %v", err)
+ return nil, err
+ }
+
+ // split namespace and subdirectory
+ pathParts := strings.Split(strings.Trim(repoPath, "/"), "/")
+ var resp *gitlab.Response
+ u, _ := url.Parse(baseURL)
+ for len(pathParts) >= 2 {
+ _, resp, err = gitlabClient.Version.GetVersion()
+ if err == nil || resp != nil && resp.StatusCode == 401 {
+ err = nil // if no authentication given, this still should work
+ break
+ }
+
+ u.Path = path.Join(u.Path, pathParts[0])
+ baseURL = u.String()
+ pathParts = pathParts[1:]
+ _ = gitlab.WithBaseURL(baseURL)(gitlabClient)
+ repoPath = strings.Join(pathParts, "/")
+ }
+ if err != nil {
+ log.Trace("Error could not get gitlab version: %v", err)
+ return nil, err
+ }
+
+ log.Trace("gitlab downloader: use BaseURL: '%s' and RepoPath: '%s'", baseURL, repoPath)
+
+ // Grab and store project/repo ID here, due to issues using the URL escaped path
+ gr, _, err := gitlabClient.Projects.GetProject(repoPath, nil, nil, gitlab.WithContext(ctx))
+ if err != nil {
+ log.Trace("Error retrieving project: %v", err)
+ return nil, err
+ }
+
+ if gr == nil {
+ log.Trace("Error getting project, project is nil")
+ return nil, errors.New("Error getting project, project is nil")
+ }
+
+ return &GitlabDownloader{
+ ctx: ctx,
+ client: gitlabClient,
+ repoID: gr.ID,
+ repoName: gr.Name,
+ maxPerPage: 100,
+ }, nil
+}
+
+// SetContext set context
+func (g *GitlabDownloader) SetContext(ctx context.Context) {
+ g.ctx = ctx
+}
+
+// GetRepoInfo returns a repository information
+func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) {
+ gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, err
+ }
+
+ var private bool
+ switch gr.Visibility {
+ case gitlab.InternalVisibility:
+ private = true
+ case gitlab.PrivateVisibility:
+ private = true
+ }
+
+ var owner string
+ if gr.Owner == nil {
+ log.Trace("gr.Owner is nil, trying to get owner from Namespace")
+ if gr.Namespace != nil && gr.Namespace.Kind == "user" {
+ owner = gr.Namespace.Path
+ }
+ } else {
+ owner = gr.Owner.Username
+ }
+
+ // convert gitlab repo to stand Repo
+ return &base.Repository{
+ Owner: owner,
+ Name: gr.Name,
+ IsPrivate: private,
+ Description: gr.Description,
+ OriginalURL: gr.WebURL,
+ CloneURL: gr.HTTPURLToRepo,
+ DefaultBranch: gr.DefaultBranch,
+ }, nil
+}
+
+// GetTopics return gitlab topics
+func (g *GitlabDownloader) GetTopics() ([]string, error) {
+ gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, err
+ }
+ return gr.TagList, err
+}
+
+// GetMilestones returns milestones
+func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
+ var perPage = g.maxPerPage
+ var state = "all"
+ var milestones = make([]*base.Milestone, 0, perPage)
+ for i := 1; ; i++ {
+ ms, _, err := g.client.Milestones.ListMilestones(g.repoID, &gitlab.ListMilestonesOptions{
+ State: &state,
+ ListOptions: gitlab.ListOptions{
+ Page: i,
+ PerPage: perPage,
+ }}, nil, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, err
+ }
+
+ for _, m := range ms {
+ var desc string
+ if m.Description != "" {
+ desc = m.Description
+ }
+ var state = "open"
+ var closedAt *time.Time
+ if m.State != "" {
+ state = m.State
+ if state == "closed" {
+ closedAt = m.UpdatedAt
+ }
+ }
+
+ var deadline *time.Time
+ if m.DueDate != nil {
+ deadlineParsed, err := time.Parse("2006-01-02", m.DueDate.String())
+ if err != nil {
+ log.Trace("Error parsing Milestone DueDate time")
+ deadline = nil
+ } else {
+ deadline = &deadlineParsed
+ }
+ }
+
+ milestones = append(milestones, &base.Milestone{
+ Title: m.Title,
+ Description: desc,
+ Deadline: deadline,
+ State: state,
+ Created: *m.CreatedAt,
+ Updated: m.UpdatedAt,
+ Closed: closedAt,
+ })
+ }
+ if len(ms) < perPage {
+ break
+ }
+ }
+ return milestones, nil
+}
+
+func (g *GitlabDownloader) normalizeColor(val string) string {
+ val = strings.TrimLeft(val, "#")
+ val = strings.ToLower(val)
+ if len(val) == 3 {
+ c := []rune(val)
+ val = fmt.Sprintf("%c%c%c%c%c%c", c[0], c[0], c[1], c[1], c[2], c[2])
+ }
+ if len(val) != 6 {
+ return ""
+ }
+ return val
+}
+
+// GetLabels returns labels
+func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
+ var perPage = g.maxPerPage
+ var labels = make([]*base.Label, 0, perPage)
+ for i := 1; ; i++ {
+ ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{
+ Page: i,
+ PerPage: perPage,
+ }}, nil, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, err
+ }
+ for _, label := range ls {
+ baseLabel := &base.Label{
+ Name: label.Name,
+ Color: g.normalizeColor(label.Color),
+ Description: label.Description,
+ }
+ labels = append(labels, baseLabel)
+ }
+ if len(ls) < perPage {
+ break
+ }
+ }
+ return labels, nil
+}
+
+func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release {
+ var zero int
+ r := &base.Release{
+ TagName: rel.TagName,
+ TargetCommitish: rel.Commit.ID,
+ Name: rel.Name,
+ Body: rel.Description,
+ Created: *rel.CreatedAt,
+ PublisherID: int64(rel.Author.ID),
+ PublisherName: rel.Author.Username,
+ }
+
+ httpClient := &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }
+
+ for k, asset := range rel.Assets.Links {
+ r.Assets = append(r.Assets, &base.ReleaseAsset{
+ ID: int64(asset.ID),
+ Name: asset.Name,
+ ContentType: &rel.Assets.Sources[k].Format,
+ Size: &zero,
+ DownloadCount: &zero,
+ DownloadFunc: func() (io.ReadCloser, error) {
+ link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, asset.ID, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, err
+ }
+
+ req, err := http.NewRequest("GET", link.URL, nil)
+ if err != nil {
+ return nil, err
+ }
+ req = req.WithContext(g.ctx)
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ // resp.Body is closed by the uploader
+ return resp.Body, nil
+ },
+ })
+ }
+ return r
+}
+
+// GetReleases returns releases
+func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
+ var perPage = g.maxPerPage
+ var releases = make([]*base.Release, 0, perPage)
+ for i := 1; ; i++ {
+ ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{
+ Page: i,
+ PerPage: perPage,
+ }, nil, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, err
+ }
+
+ for _, release := range ls {
+ releases = append(releases, g.convertGitlabRelease(release))
+ }
+ if len(ls) < perPage {
+ break
+ }
+ }
+ return releases, nil
+}
+
+type gitlabIssueContext struct {
+ foreignID int64
+ localID int64
+ IsMergeRequest bool
+}
+
+func (c gitlabIssueContext) LocalID() int64 {
+ return c.localID
+}
+
+func (c gitlabIssueContext) ForeignID() int64 {
+ return c.foreignID
+}
+
+// GetIssues returns issues according start and limit
+// Note: issue label description and colors are not supported by the go-gitlab library at this time
+func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
+ state := "all"
+ sort := "asc"
+
+ if perPage > g.maxPerPage {
+ perPage = g.maxPerPage
+ }
+
+ opt := &gitlab.ListProjectIssuesOptions{
+ State: &state,
+ Sort: &sort,
+ ListOptions: gitlab.ListOptions{
+ PerPage: perPage,
+ Page: page,
+ },
+ }
+
+ var allIssues = make([]*base.Issue, 0, perPage)
+
+ issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing issues: %v", err)
+ }
+ for _, issue := range issues {
+
+ var labels = make([]*base.Label, 0, len(issue.Labels))
+ for _, l := range issue.Labels {
+ labels = append(labels, &base.Label{
+ Name: l,
+ })
+ }
+
+ var milestone string
+ if issue.Milestone != nil {
+ milestone = issue.Milestone.Title
+ }
+
+ var reactions []*base.Reaction
+ var awardPage = 1
+ for {
+ awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing issue awards: %v", err)
+ }
+
+ for i := range awards {
+ reactions = append(reactions, g.awardToReaction(awards[i]))
+ }
+
+ if len(awards) < perPage {
+ break
+ }
+
+ awardPage++
+ }
+
+ allIssues = append(allIssues, &base.Issue{
+ Title: issue.Title,
+ Number: int64(issue.IID),
+ PosterID: int64(issue.Author.ID),
+ PosterName: issue.Author.Username,
+ Content: issue.Description,
+ Milestone: milestone,
+ State: issue.State,
+ Created: *issue.CreatedAt,
+ Labels: labels,
+ Reactions: reactions,
+ Closed: issue.ClosedAt,
+ IsLocked: issue.DiscussionLocked,
+ Updated: *issue.UpdatedAt,
+ Context: gitlabIssueContext{
+ foreignID: int64(issue.IID),
+ localID: int64(issue.IID),
+ IsMergeRequest: false,
+ },
+ })
+
+ // increment issueCount, to be used in GetPullRequests()
+ g.issueCount++
+ }
+
+ return allIssues, len(issues) < perPage, nil
+}
+
+// GetComments returns comments according issueNumber
+// TODO: figure out how to transfer comment reactions
+func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
+ context, ok := opts.Context.(gitlabIssueContext)
+ if !ok {
+ return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context)
+ }
+
+ var allComments = make([]*base.Comment, 0, g.maxPerPage)
+
+ var page = 1
+
+ for {
+ var comments []*gitlab.Discussion
+ var resp *gitlab.Response
+ var err error
+ if !context.IsMergeRequest {
+ comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListIssueDiscussionsOptions{
+ Page: page,
+ PerPage: g.maxPerPage,
+ }, nil, gitlab.WithContext(g.ctx))
+ } else {
+ comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListMergeRequestDiscussionsOptions{
+ Page: page,
+ PerPage: g.maxPerPage,
+ }, nil, gitlab.WithContext(g.ctx))
+ }
+
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing comments: %v %v", g.repoID, err)
+ }
+ for _, comment := range comments {
+ // Flatten comment threads
+ if !comment.IndividualNote {
+ for _, note := range comment.Notes {
+ allComments = append(allComments, &base.Comment{
+ IssueIndex: context.LocalID(),
+ PosterID: int64(note.Author.ID),
+ PosterName: note.Author.Username,
+ PosterEmail: note.Author.Email,
+ Content: note.Body,
+ Created: *note.CreatedAt,
+ })
+ }
+ } else {
+ c := comment.Notes[0]
+ allComments = append(allComments, &base.Comment{
+ IssueIndex: context.LocalID(),
+ PosterID: int64(c.Author.ID),
+ PosterName: c.Author.Username,
+ PosterEmail: c.Author.Email,
+ Content: c.Body,
+ Created: *c.CreatedAt,
+ })
+ }
+
+ }
+ if resp.NextPage == 0 {
+ break
+ }
+ page = resp.NextPage
+ }
+ return allComments, true, nil
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
+ if perPage > g.maxPerPage {
+ perPage = g.maxPerPage
+ }
+
+ opt := &gitlab.ListProjectMergeRequestsOptions{
+ ListOptions: gitlab.ListOptions{
+ PerPage: perPage,
+ Page: page,
+ },
+ }
+
+ var allPRs = make([]*base.PullRequest, 0, perPage)
+
+ prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing merge requests: %v", err)
+ }
+ for _, pr := range prs {
+
+ var labels = make([]*base.Label, 0, len(pr.Labels))
+ for _, l := range pr.Labels {
+ labels = append(labels, &base.Label{
+ Name: l,
+ })
+ }
+
+ var merged bool
+ if pr.State == "merged" {
+ merged = true
+ pr.State = "closed"
+ }
+
+ var mergeTime = pr.MergedAt
+ if merged && pr.MergedAt == nil {
+ mergeTime = pr.UpdatedAt
+ }
+
+ var closeTime = pr.ClosedAt
+ if merged && pr.ClosedAt == nil {
+ closeTime = pr.UpdatedAt
+ }
+
+ var locked bool
+ if pr.State == "locked" {
+ locked = true
+ }
+
+ var milestone string
+ if pr.Milestone != nil {
+ milestone = pr.Milestone.Title
+ }
+
+ var reactions []*base.Reaction
+ var awardPage = 1
+ for {
+ awards, _, err := g.client.AwardEmoji.ListMergeRequestAwardEmoji(g.repoID, pr.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing merge requests awards: %v", err)
+ }
+
+ for i := range awards {
+ reactions = append(reactions, g.awardToReaction(awards[i]))
+ }
+
+ if len(awards) < perPage {
+ break
+ }
+
+ awardPage++
+ }
+
+ // Add the PR ID to the Issue Count because PR and Issues share ID space in Gitea
+ newPRNumber := g.issueCount + int64(pr.IID)
+
+ allPRs = append(allPRs, &base.PullRequest{
+ Title: pr.Title,
+ Number: newPRNumber,
+ PosterName: pr.Author.Username,
+ PosterID: int64(pr.Author.ID),
+ Content: pr.Description,
+ Milestone: milestone,
+ State: pr.State,
+ Created: *pr.CreatedAt,
+ Closed: closeTime,
+ Labels: labels,
+ Merged: merged,
+ MergeCommitSHA: pr.MergeCommitSHA,
+ MergedTime: mergeTime,
+ IsLocked: locked,
+ Reactions: reactions,
+ Head: base.PullRequestBranch{
+ Ref: pr.SourceBranch,
+ SHA: pr.SHA,
+ RepoName: g.repoName,
+ OwnerName: pr.Author.Username,
+ CloneURL: pr.WebURL,
+ },
+ Base: base.PullRequestBranch{
+ Ref: pr.TargetBranch,
+ SHA: pr.DiffRefs.BaseSha,
+ RepoName: g.repoName,
+ OwnerName: pr.Author.Username,
+ },
+ PatchURL: pr.WebURL + ".patch",
+ Context: gitlabIssueContext{
+ foreignID: int64(pr.IID),
+ localID: newPRNumber,
+ IsMergeRequest: true,
+ },
+ })
+ }
+
+ return allPRs, len(prs) < perPage, nil
+}
+
+// GetReviews returns pull requests review
+func (g *GitlabDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
+ approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(context.ForeignID()), gitlab.WithContext(g.ctx))
+ if err != nil {
+ if resp != nil && resp.StatusCode == 404 {
+ log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error()))
+ return []*base.Review{}, nil
+ }
+ return nil, err
+ }
+
+ var reviews = make([]*base.Review, 0, len(approvals.ApprovedBy))
+ for _, user := range approvals.ApprovedBy {
+ reviews = append(reviews, &base.Review{
+ IssueIndex: context.LocalID(),
+ ReviewerID: int64(user.User.ID),
+ ReviewerName: user.User.Username,
+ CreatedAt: *approvals.UpdatedAt,
+ // All we get are approvals
+ State: base.ReviewStateApproved,
+ })
+ }
+
+ return reviews, nil
+}
+
+func (g *GitlabDownloader) awardToReaction(award *gitlab.AwardEmoji) *base.Reaction {
+ return &base.Reaction{
+ UserID: int64(award.User.ID),
+ UserName: award.User.Username,
+ Content: award.Name,
+ }
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "os"
+ "testing"
+ "time"
+
+ base "code.gitea.io/gitea/modules/migration"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGitlabDownloadRepo(t *testing.T) {
+ // Skip tests if Gitlab token is not found
+ gitlabPersonalAccessToken := os.Getenv("GITLAB_READ_TOKEN")
+ if gitlabPersonalAccessToken == "" {
+ t.Skip("skipped test because GITLAB_READ_TOKEN was not in the environment")
+ }
+
+ resp, err := http.Get("https://gitlab.com/gitea/test_repo")
+ if err != nil || resp.StatusCode != 200 {
+ t.Skipf("Can't access test repo, skipping %s", t.Name())
+ }
+
+ downloader, err := NewGitlabDownloader(context.Background(), "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken)
+ if err != nil {
+ t.Fatal(fmt.Sprintf("NewGitlabDownloader is nil: %v", err))
+ }
+ repo, err := downloader.GetRepoInfo()
+ assert.NoError(t, err)
+ // Repo Owner is blank in Gitlab Group repos
+ assertRepositoryEqual(t, &base.Repository{
+ Name: "test_repo",
+ Owner: "",
+ Description: "Test repository for testing migration from gitlab to gitea",
+ CloneURL: "https://gitlab.com/gitea/test_repo.git",
+ OriginalURL: "https://gitlab.com/gitea/test_repo",
+ DefaultBranch: "master",
+ }, repo)
+
+ topics, err := downloader.GetTopics()
+ assert.NoError(t, err)
+ assert.True(t, len(topics) == 2)
+ assert.EqualValues(t, []string{"migration", "test"}, topics)
+
+ milestones, err := downloader.GetMilestones()
+ assert.NoError(t, err)
+ assertMilestonesEqual(t, []*base.Milestone{
+ {
+ Title: "1.1.0",
+ Created: time.Date(2019, 11, 28, 8, 42, 44, 575000000, time.UTC),
+ Updated: timePtr(time.Date(2019, 11, 28, 8, 42, 44, 575000000, time.UTC)),
+ State: "active",
+ },
+ {
+ Title: "1.0.0",
+ Created: time.Date(2019, 11, 28, 8, 42, 30, 301000000, time.UTC),
+ Updated: timePtr(time.Date(2019, 11, 28, 15, 57, 52, 401000000, time.UTC)),
+ Closed: timePtr(time.Date(2019, 11, 28, 15, 57, 52, 401000000, time.UTC)),
+ State: "closed",
+ },
+ }, milestones)
+
+ labels, err := downloader.GetLabels()
+ assert.NoError(t, err)
+ assertLabelsEqual(t, []*base.Label{
+ {
+ Name: "bug",
+ Color: "d9534f",
+ },
+ {
+ Name: "confirmed",
+ Color: "d9534f",
+ },
+ {
+ Name: "critical",
+ Color: "d9534f",
+ },
+ {
+ Name: "discussion",
+ Color: "428bca",
+ },
+ {
+ Name: "documentation",
+ Color: "f0ad4e",
+ },
+ {
+ Name: "duplicate",
+ Color: "7f8c8d",
+ },
+ {
+ Name: "enhancement",
+ Color: "5cb85c",
+ },
+ {
+ Name: "suggestion",
+ Color: "428bca",
+ },
+ {
+ Name: "support",
+ Color: "f0ad4e",
+ },
+ }, labels)
+
+ releases, err := downloader.GetReleases()
+ assert.NoError(t, err)
+ assertReleasesEqual(t, []*base.Release{
+ {
+ TagName: "v0.9.99",
+ TargetCommitish: "0720a3ec57c1f843568298117b874319e7deee75",
+ Name: "First Release",
+ Body: "A test release",
+ Created: time.Date(2019, 11, 28, 9, 9, 48, 840000000, time.UTC),
+ PublisherID: 1241334,
+ PublisherName: "lafriks",
+ },
+ }, releases)
+
+ issues, isEnd, err := downloader.GetIssues(1, 2)
+ assert.NoError(t, err)
+ assert.False(t, isEnd)
+
+ assertIssuesEqual(t, []*base.Issue{
+ {
+ Number: 1,
+ Title: "Please add an animated gif icon to the merge button",
+ Content: "I just want the merge button to hurt my eyes a little. :stuck_out_tongue_closed_eyes:",
+ Milestone: "1.0.0",
+ PosterID: 1241334,
+ PosterName: "lafriks",
+ State: "closed",
+ Created: time.Date(2019, 11, 28, 8, 43, 35, 459000000, time.UTC),
+ Updated: time.Date(2019, 11, 28, 8, 46, 23, 304000000, time.UTC),
+ Labels: []*base.Label{
+ {
+ Name: "bug",
+ },
+ {
+ Name: "discussion",
+ },
+ },
+ Reactions: []*base.Reaction{
+ {
+ UserID: 1241334,
+ UserName: "lafriks",
+ Content: "thumbsup",
+ },
+ {
+ UserID: 1241334,
+ UserName: "lafriks",
+ Content: "open_mouth",
+ }},
+ Closed: timePtr(time.Date(2019, 11, 28, 8, 46, 23, 275000000, time.UTC)),
+ },
+ {
+ Number: 2,
+ Title: "Test issue",
+ Content: "This is test issue 2, do not touch!",
+ Milestone: "1.1.0",
+ PosterID: 1241334,
+ PosterName: "lafriks",
+ State: "closed",
+ Created: time.Date(2019, 11, 28, 8, 44, 46, 277000000, time.UTC),
+ Updated: time.Date(2019, 11, 28, 8, 45, 44, 987000000, time.UTC),
+ Labels: []*base.Label{
+ {
+ Name: "duplicate",
+ },
+ },
+ Reactions: []*base.Reaction{
+ {
+ UserID: 1241334,
+ UserName: "lafriks",
+ Content: "thumbsup",
+ },
+ {
+ UserID: 1241334,
+ UserName: "lafriks",
+ Content: "thumbsdown",
+ },
+ {
+ UserID: 1241334,
+ UserName: "lafriks",
+ Content: "laughing",
+ },
+ {
+ UserID: 1241334,
+ UserName: "lafriks",
+ Content: "tada",
+ },
+ {
+ UserID: 1241334,
+ UserName: "lafriks",
+ Content: "confused",
+ },
+ {
+ UserID: 1241334,
+ UserName: "lafriks",
+ Content: "hearts",
+ }},
+ Closed: timePtr(time.Date(2019, 11, 28, 8, 45, 44, 959000000, time.UTC)),
+ },
+ }, issues)
+
+ comments, _, err := downloader.GetComments(base.GetCommentOptions{
+ Context: gitlabIssueContext{
+ foreignID: 2,
+ localID: 2,
+ IsMergeRequest: false,
+ },
+ })
+ assert.NoError(t, err)
+ assertCommentsEqual(t, []*base.Comment{
+ {
+ IssueIndex: 2,
+ PosterID: 1241334,
+ PosterName: "lafriks",
+ Created: time.Date(2019, 11, 28, 8, 44, 52, 501000000, time.UTC),
+ Content: "This is a comment",
+ Reactions: nil,
+ },
+ {
+ IssueIndex: 2,
+ PosterID: 1241334,
+ PosterName: "lafriks",
+ Created: time.Date(2019, 11, 28, 8, 45, 2, 329000000, time.UTC),
+ Content: "changed milestone to %2",
+ Reactions: nil,
+ },
+ {
+ IssueIndex: 2,
+ PosterID: 1241334,
+ PosterName: "lafriks",
+ Created: time.Date(2019, 11, 28, 8, 45, 45, 7000000, time.UTC),
+ Content: "closed",
+ Reactions: nil,
+ },
+ {
+ IssueIndex: 2,
+ PosterID: 1241334,
+ PosterName: "lafriks",
+ Created: time.Date(2019, 11, 28, 8, 45, 53, 501000000, time.UTC),
+ Content: "A second comment",
+ Reactions: nil,
+ },
+ }, comments)
+
+ prs, _, err := downloader.GetPullRequests(1, 1)
+ assert.NoError(t, err)
+ assertPullRequestsEqual(t, []*base.PullRequest{
+ {
+ Number: 4,
+ Title: "Test branch",
+ Content: "do not merge this PR",
+ Milestone: "1.0.0",
+ PosterID: 1241334,
+ PosterName: "lafriks",
+ State: "opened",
+ Created: time.Date(2019, 11, 28, 15, 56, 54, 104000000, time.UTC),
+ Labels: []*base.Label{
+ {
+ Name: "bug",
+ },
+ },
+ Reactions: []*base.Reaction{{
+ UserID: 4575606,
+ UserName: "real6543",
+ Content: "thumbsup",
+ }, {
+ UserID: 4575606,
+ UserName: "real6543",
+ Content: "tada",
+ }},
+ PatchURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2.patch",
+ Head: base.PullRequestBranch{
+ Ref: "feat/test",
+ CloneURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2",
+ SHA: "9f733b96b98a4175276edf6a2e1231489c3bdd23",
+ RepoName: "test_repo",
+ OwnerName: "lafriks",
+ },
+ Base: base.PullRequestBranch{
+ Ref: "master",
+ SHA: "",
+ OwnerName: "lafriks",
+ RepoName: "test_repo",
+ },
+ Closed: nil,
+ Merged: false,
+ MergedTime: nil,
+ MergeCommitSHA: "",
+ Context: gitlabIssueContext{
+ foreignID: 2,
+ localID: 4,
+ IsMergeRequest: true,
+ },
+ },
+ }, prs)
+
+ rvs, err := downloader.GetReviews(base.BasicIssueContext(1))
+ assert.NoError(t, err)
+ assertReviewsEqual(t, []*base.Review{
+ {
+ ReviewerID: 4102996,
+ ReviewerName: "zeripath",
+ CreatedAt: time.Date(2019, 11, 28, 16, 02, 8, 377000000, time.UTC),
+ State: "APPROVED",
+ },
+ {
+ ReviewerID: 527793,
+ ReviewerName: "axifive",
+ CreatedAt: time.Date(2019, 11, 28, 16, 02, 8, 377000000, time.UTC),
+ State: "APPROVED",
+ },
+ }, rvs)
+
+ rvs, err = downloader.GetReviews(base.BasicIssueContext(2))
+ assert.NoError(t, err)
+ assertReviewsEqual(t, []*base.Review{
+ {
+ ReviewerID: 4575606,
+ ReviewerName: "real6543",
+ CreatedAt: time.Date(2020, 04, 19, 19, 24, 21, 108000000, time.UTC),
+ State: "APPROVED",
+ },
+ }, rvs)
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "crypto/tls"
+ "fmt"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/modules/log"
+ base "code.gitea.io/gitea/modules/migration"
+ "code.gitea.io/gitea/modules/proxy"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/structs"
+
+ "github.com/gogs/go-gogs-client"
+)
+
+var (
+ _ base.Downloader = &GogsDownloader{}
+ _ base.DownloaderFactory = &GogsDownloaderFactory{}
+)
+
+func init() {
+ RegisterDownloaderFactory(&GogsDownloaderFactory{})
+}
+
+// GogsDownloaderFactory defines a gogs downloader factory
+type GogsDownloaderFactory struct {
+}
+
+// New returns a Downloader related to this factory according MigrateOptions
+func (f *GogsDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
+ u, err := url.Parse(opts.CloneAddr)
+ if err != nil {
+ return nil, err
+ }
+
+ baseURL := u.Scheme + "://" + u.Host
+ repoNameSpace := strings.TrimSuffix(u.Path, ".git")
+ repoNameSpace = strings.Trim(repoNameSpace, "/")
+
+ fields := strings.Split(repoNameSpace, "/")
+ if len(fields) < 2 {
+ return nil, fmt.Errorf("invalid path: %s", repoNameSpace)
+ }
+
+ log.Trace("Create gogs downloader. BaseURL: %s RepoOwner: %s RepoName: %s", baseURL, fields[0], fields[1])
+ return NewGogsDownloader(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, fields[0], fields[1]), nil
+}
+
+// GitServiceType returns the type of git service
+func (f *GogsDownloaderFactory) GitServiceType() structs.GitServiceType {
+ return structs.GogsService
+}
+
+// GogsDownloader implements a Downloader interface to get repository information
+// from gogs via API
+type GogsDownloader struct {
+ base.NullDownloader
+ ctx context.Context
+ client *gogs.Client
+ baseURL string
+ repoOwner string
+ repoName string
+ userName string
+ password string
+ openIssuesFinished bool
+ openIssuesPages int
+ transport http.RoundTripper
+}
+
+// SetContext set context
+func (g *GogsDownloader) SetContext(ctx context.Context) {
+ g.ctx = ctx
+}
+
+// NewGogsDownloader creates a gogs Downloader via gogs API
+func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader {
+ var downloader = GogsDownloader{
+ ctx: ctx,
+ baseURL: baseURL,
+ userName: userName,
+ password: password,
+ repoOwner: repoOwner,
+ repoName: repoName,
+ }
+
+ var client *gogs.Client
+ if len(token) != 0 {
+ client = gogs.NewClient(baseURL, token)
+ downloader.userName = token
+ } else {
+ downloader.transport = &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: func(req *http.Request) (*url.URL, error) {
+ req.SetBasicAuth(userName, password)
+ return proxy.Proxy()(req)
+ },
+ }
+
+ client = gogs.NewClient(baseURL, "")
+ client.SetHTTPClient(&http.Client{
+ Transport: &downloader,
+ })
+ }
+
+ downloader.client = client
+ return &downloader
+}
+
+// RoundTrip wraps the provided request within this downloader's context and passes it to our internal http.Transport.
+// This implements http.RoundTripper and makes the gogs client requests cancellable even though it is not cancellable itself
+func (g *GogsDownloader) RoundTrip(req *http.Request) (*http.Response, error) {
+ return g.transport.RoundTrip(req.WithContext(g.ctx))
+}
+
+// GetRepoInfo returns a repository information
+func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) {
+ gr, err := g.client.GetRepo(g.repoOwner, g.repoName)
+ if err != nil {
+ return nil, err
+ }
+
+ // convert gogs repo to stand Repo
+ return &base.Repository{
+ Owner: g.repoOwner,
+ Name: g.repoName,
+ IsPrivate: gr.Private,
+ Description: gr.Description,
+ CloneURL: gr.CloneURL,
+ OriginalURL: gr.HTMLURL,
+ DefaultBranch: gr.DefaultBranch,
+ }, nil
+}
+
+// GetMilestones returns milestones
+func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) {
+ var perPage = 100
+ var milestones = make([]*base.Milestone, 0, perPage)
+
+ ms, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, m := range ms {
+ milestones = append(milestones, &base.Milestone{
+ Title: m.Title,
+ Description: m.Description,
+ Deadline: m.Deadline,
+ State: string(m.State),
+ Closed: m.Closed,
+ })
+ }
+
+ return milestones, nil
+}
+
+// GetLabels returns labels
+func (g *GogsDownloader) GetLabels() ([]*base.Label, error) {
+ var perPage = 100
+ var labels = make([]*base.Label, 0, perPage)
+ ls, err := g.client.ListRepoLabels(g.repoOwner, g.repoName)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, label := range ls {
+ labels = append(labels, convertGogsLabel(label))
+ }
+
+ return labels, nil
+}
+
+// GetIssues returns issues according start and limit, perPage is not supported
+func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) {
+ var state string
+ if g.openIssuesFinished {
+ state = string(gogs.STATE_CLOSED)
+ page -= g.openIssuesPages
+ } else {
+ state = string(gogs.STATE_OPEN)
+ g.openIssuesPages = page
+ }
+
+ issues, isEnd, err := g.getIssues(page, state)
+ if err != nil {
+ return nil, false, err
+ }
+
+ if isEnd {
+ if g.openIssuesFinished {
+ return issues, true, nil
+ }
+ g.openIssuesFinished = true
+ }
+
+ return issues, false, nil
+}
+
+func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, error) {
+ var allIssues = make([]*base.Issue, 0, 10)
+
+ issues, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{
+ Page: page,
+ State: state,
+ })
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing repos: %v", err)
+ }
+
+ for _, issue := range issues {
+ if issue.PullRequest != nil {
+ continue
+ }
+ allIssues = append(allIssues, convertGogsIssue(issue))
+ }
+
+ return allIssues, len(issues) == 0, nil
+}
+
+// GetComments returns comments according issueNumber
+func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
+ var allComments = make([]*base.Comment, 0, 100)
+
+ comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID())
+ if err != nil {
+ return nil, false, fmt.Errorf("error while listing repos: %v", err)
+ }
+ for _, comment := range comments {
+ if len(comment.Body) == 0 || comment.Poster == nil {
+ continue
+ }
+ allComments = append(allComments, &base.Comment{
+ IssueIndex: opts.Context.LocalID(),
+ PosterID: comment.Poster.ID,
+ PosterName: comment.Poster.Login,
+ PosterEmail: comment.Poster.Email,
+ Content: comment.Body,
+ Created: comment.Created,
+ Updated: comment.Updated,
+ })
+ }
+
+ return allComments, true, nil
+}
+
+// GetTopics return repository topics
+func (g *GogsDownloader) GetTopics() ([]string, error) {
+ return []string{}, nil
+}
+
+// FormatCloneURL add authentification into remote URLs
+func (g *GogsDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
+ if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
+ u, err := url.Parse(remoteAddr)
+ if err != nil {
+ return "", err
+ }
+ if len(opts.AuthToken) != 0 {
+ u.User = url.UserPassword(opts.AuthToken, "")
+ } else {
+ u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
+ }
+ return u.String(), nil
+ }
+ return remoteAddr, nil
+}
+
+func convertGogsIssue(issue *gogs.Issue) *base.Issue {
+ var milestone string
+ if issue.Milestone != nil {
+ milestone = issue.Milestone.Title
+ }
+ var labels = make([]*base.Label, 0, len(issue.Labels))
+ for _, l := range issue.Labels {
+ labels = append(labels, convertGogsLabel(l))
+ }
+
+ var closed *time.Time
+ if issue.State == gogs.STATE_CLOSED {
+ // gogs client haven't provide closed, so we use updated instead
+ closed = &issue.Updated
+ }
+
+ return &base.Issue{
+ Title: issue.Title,
+ Number: issue.Index,
+ PosterID: issue.Poster.ID,
+ PosterName: issue.Poster.Login,
+ PosterEmail: issue.Poster.Email,
+ Content: issue.Body,
+ Milestone: milestone,
+ State: string(issue.State),
+ Created: issue.Created,
+ Updated: issue.Updated,
+ Labels: labels,
+ Closed: closed,
+ Context: base.BasicIssueContext(issue.Index),
+ }
+}
+
+func convertGogsLabel(label *gogs.Label) *base.Label {
+ return &base.Label{
+ Name: label.Name,
+ Color: label.Color,
+ }
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "net/http"
+ "os"
+ "testing"
+ "time"
+
+ base "code.gitea.io/gitea/modules/migration"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGogsDownloadRepo(t *testing.T) {
+ // Skip tests if Gogs token is not found
+ gogsPersonalAccessToken := os.Getenv("GOGS_READ_TOKEN")
+ if len(gogsPersonalAccessToken) == 0 {
+ t.Skip("skipped test because GOGS_READ_TOKEN was not in the environment")
+ }
+
+ resp, err := http.Get("https://try.gogs.io/lunnytest/TESTREPO")
+ if err != nil || resp.StatusCode/100 != 2 {
+ // skip and don't run test
+ t.Skipf("visit test repo failed, ignored")
+ return
+ }
+
+ downloader := NewGogsDownloader(context.Background(), "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO")
+ repo, err := downloader.GetRepoInfo()
+ assert.NoError(t, err)
+
+ assertRepositoryEqual(t, &base.Repository{
+ Name: "TESTREPO",
+ Owner: "lunnytest",
+ Description: "",
+ CloneURL: "https://try.gogs.io/lunnytest/TESTREPO.git",
+ OriginalURL: "https://try.gogs.io/lunnytest/TESTREPO",
+ DefaultBranch: "master",
+ }, repo)
+
+ milestones, err := downloader.GetMilestones()
+ assert.NoError(t, err)
+ assertMilestonesEqual(t, []*base.Milestone{
+ {
+ Title: "1.0",
+ State: "open",
+ },
+ }, milestones)
+
+ labels, err := downloader.GetLabels()
+ assert.NoError(t, err)
+ assertLabelsEqual(t, []*base.Label{
+ {
+ Name: "bug",
+ Color: "ee0701",
+ },
+ {
+ Name: "duplicate",
+ Color: "cccccc",
+ },
+ {
+ Name: "enhancement",
+ Color: "84b6eb",
+ },
+ {
+ Name: "help wanted",
+ Color: "128a0c",
+ },
+ {
+ Name: "invalid",
+ Color: "e6e6e6",
+ },
+ {
+ Name: "question",
+ Color: "cc317c",
+ },
+ {
+ Name: "wontfix",
+ Color: "ffffff",
+ },
+ }, labels)
+
+ // downloader.GetIssues()
+ issues, isEnd, err := downloader.GetIssues(1, 8)
+ assert.NoError(t, err)
+ assert.False(t, isEnd)
+ assertIssuesEqual(t, []*base.Issue{
+ {
+ Number: 1,
+ PosterID: 5331,
+ PosterName: "lunny",
+ PosterEmail: "xiaolunwen@gmail.com",
+ Title: "test",
+ Content: "test",
+ Milestone: "",
+ State: "open",
+ Created: time.Date(2019, 06, 11, 8, 16, 44, 0, time.UTC),
+ Updated: time.Date(2019, 10, 26, 11, 07, 2, 0, time.UTC),
+ Labels: []*base.Label{
+ {
+ Name: "bug",
+ Color: "ee0701",
+ },
+ },
+ },
+ }, issues)
+
+ // downloader.GetComments()
+ comments, _, err := downloader.GetComments(base.GetCommentOptions{
+ Context: base.BasicIssueContext(1),
+ })
+ assert.NoError(t, err)
+ assertCommentsEqual(t, []*base.Comment{
+ {
+ IssueIndex: 1,
+ PosterID: 5331,
+ PosterName: "lunny",
+ PosterEmail: "xiaolunwen@gmail.com",
+ Created: time.Date(2019, 06, 11, 8, 19, 50, 0, time.UTC),
+ Updated: time.Date(2019, 06, 11, 8, 19, 50, 0, time.UTC),
+ Content: "1111",
+ },
+ {
+ IssueIndex: 1,
+ PosterID: 15822,
+ PosterName: "clacplouf",
+ PosterEmail: "test1234@dbn.re",
+ Created: time.Date(2019, 10, 26, 11, 7, 2, 0, time.UTC),
+ Updated: time.Date(2019, 10, 26, 11, 7, 2, 0, time.UTC),
+ Content: "88888888",
+ },
+ }, comments)
+
+ // downloader.GetPullRequests()
+ _, _, err = downloader.GetPullRequests(1, 3)
+ assert.Error(t, err)
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "path/filepath"
+ "testing"
+ "time"
+
+ "code.gitea.io/gitea/models/unittest"
+ base "code.gitea.io/gitea/modules/migration"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMain(m *testing.M) {
+ unittest.MainTest(m, filepath.Join("..", ".."))
+}
+
+func timePtr(t time.Time) *time.Time {
+ return &t
+}
+
+func assertTimeEqual(t *testing.T, expected, actual time.Time) {
+ assert.Equal(t, expected.UTC(), actual.UTC())
+}
+
+func assertTimePtrEqual(t *testing.T, expected, actual *time.Time) {
+ if expected == nil {
+ assert.Nil(t, actual)
+ } else {
+ assertTimeEqual(t, *expected, *actual)
+ }
+}
+
+func assertCommentEqual(t *testing.T, expected, actual *base.Comment) {
+ assert.Equal(t, expected.IssueIndex, actual.IssueIndex)
+ assert.Equal(t, expected.PosterID, actual.PosterID)
+ assert.Equal(t, expected.PosterName, actual.PosterName)
+ assert.Equal(t, expected.PosterEmail, actual.PosterEmail)
+ assertTimeEqual(t, expected.Created, actual.Created)
+ assertTimeEqual(t, expected.Updated, actual.Updated)
+ assert.Equal(t, expected.Content, actual.Content)
+ assertReactionsEqual(t, expected.Reactions, actual.Reactions)
+}
+
+func assertCommentsEqual(t *testing.T, expected, actual []*base.Comment) {
+ if assert.Len(t, actual, len(expected)) {
+ for i := range expected {
+ assertCommentEqual(t, expected[i], actual[i])
+ }
+ }
+}
+
+func assertLabelEqual(t *testing.T, expected, actual *base.Label) {
+ assert.Equal(t, expected.Name, actual.Name)
+ assert.Equal(t, expected.Color, actual.Color)
+ assert.Equal(t, expected.Description, actual.Description)
+}
+
+func assertLabelsEqual(t *testing.T, expected, actual []*base.Label) {
+ if assert.Len(t, actual, len(expected)) {
+ for i := range expected {
+ assertLabelEqual(t, expected[i], actual[i])
+ }
+ }
+}
+
+func assertMilestoneEqual(t *testing.T, expected, actual *base.Milestone) {
+ assert.Equal(t, expected.Title, actual.Title)
+ assert.Equal(t, expected.Description, actual.Description)
+ assertTimePtrEqual(t, expected.Deadline, actual.Deadline)
+ assertTimeEqual(t, expected.Created, actual.Created)
+ assertTimePtrEqual(t, expected.Updated, actual.Updated)
+ assertTimePtrEqual(t, expected.Closed, actual.Closed)
+ assert.Equal(t, expected.State, actual.State)
+}
+
+func assertMilestonesEqual(t *testing.T, expected, actual []*base.Milestone) {
+ if assert.Len(t, actual, len(expected)) {
+ for i := range expected {
+ assertMilestoneEqual(t, expected[i], actual[i])
+ }
+ }
+}
+
+func assertIssueEqual(t *testing.T, expected, actual *base.Issue) {
+ assert.Equal(t, expected.Number, actual.Number)
+ assert.Equal(t, expected.PosterID, actual.PosterID)
+ assert.Equal(t, expected.PosterName, actual.PosterName)
+ assert.Equal(t, expected.PosterEmail, actual.PosterEmail)
+ assert.Equal(t, expected.Title, actual.Title)
+ assert.Equal(t, expected.Content, actual.Content)
+ assert.Equal(t, expected.Ref, actual.Ref)
+ assert.Equal(t, expected.Milestone, actual.Milestone)
+ assert.Equal(t, expected.State, actual.State)
+ assert.Equal(t, expected.IsLocked, actual.IsLocked)
+ assertTimeEqual(t, expected.Created, actual.Created)
+ assertTimeEqual(t, expected.Updated, actual.Updated)
+ assertTimePtrEqual(t, expected.Closed, actual.Closed)
+ assertLabelsEqual(t, expected.Labels, actual.Labels)
+ assertReactionsEqual(t, expected.Reactions, actual.Reactions)
+ assert.ElementsMatch(t, expected.Assignees, actual.Assignees)
+}
+
+func assertIssuesEqual(t *testing.T, expected, actual []*base.Issue) {
+ if assert.Len(t, actual, len(expected)) {
+ for i := range expected {
+ assertIssueEqual(t, expected[i], actual[i])
+ }
+ }
+}
+
+func assertPullRequestEqual(t *testing.T, expected, actual *base.PullRequest) {
+ assert.Equal(t, expected.Number, actual.Number)
+ assert.Equal(t, expected.Title, actual.Title)
+ assert.Equal(t, expected.PosterID, actual.PosterID)
+ assert.Equal(t, expected.PosterName, actual.PosterName)
+ assert.Equal(t, expected.PosterEmail, actual.PosterEmail)
+ assert.Equal(t, expected.Content, actual.Content)
+ assert.Equal(t, expected.Milestone, actual.Milestone)
+ assert.Equal(t, expected.State, actual.State)
+ assertTimeEqual(t, expected.Created, actual.Created)
+ assertTimeEqual(t, expected.Updated, actual.Updated)
+ assertTimePtrEqual(t, expected.Closed, actual.Closed)
+ assertLabelsEqual(t, expected.Labels, actual.Labels)
+ assert.Equal(t, expected.PatchURL, actual.PatchURL)
+ assert.Equal(t, expected.Merged, actual.Merged)
+ assertTimePtrEqual(t, expected.MergedTime, actual.MergedTime)
+ assert.Equal(t, expected.MergeCommitSHA, actual.MergeCommitSHA)
+ assertPullRequestBranchEqual(t, expected.Head, actual.Head)
+ assertPullRequestBranchEqual(t, expected.Base, actual.Base)
+ assert.ElementsMatch(t, expected.Assignees, actual.Assignees)
+ assert.Equal(t, expected.IsLocked, actual.IsLocked)
+ assertReactionsEqual(t, expected.Reactions, actual.Reactions)
+}
+
+func assertPullRequestsEqual(t *testing.T, expected, actual []*base.PullRequest) {
+ if assert.Len(t, actual, len(expected)) {
+ for i := range expected {
+ assertPullRequestEqual(t, expected[i], actual[i])
+ }
+ }
+}
+
+func assertPullRequestBranchEqual(t *testing.T, expected, actual base.PullRequestBranch) {
+ assert.Equal(t, expected.CloneURL, actual.CloneURL)
+ assert.Equal(t, expected.Ref, actual.Ref)
+ assert.Equal(t, expected.SHA, actual.SHA)
+ assert.Equal(t, expected.RepoName, actual.RepoName)
+ assert.Equal(t, expected.OwnerName, actual.OwnerName)
+}
+
+func assertReactionEqual(t *testing.T, expected, actual *base.Reaction) {
+ assert.Equal(t, expected.UserID, actual.UserID)
+ assert.Equal(t, expected.UserName, actual.UserName)
+ assert.Equal(t, expected.Content, actual.Content)
+}
+
+func assertReactionsEqual(t *testing.T, expected, actual []*base.Reaction) {
+ if assert.Len(t, actual, len(expected)) {
+ for i := range expected {
+ assertReactionEqual(t, expected[i], actual[i])
+ }
+ }
+}
+
+func assertReleaseAssetEqual(t *testing.T, expected, actual *base.ReleaseAsset) {
+ assert.Equal(t, expected.ID, actual.ID)
+ assert.Equal(t, expected.Name, actual.Name)
+ assert.Equal(t, expected.ContentType, actual.ContentType)
+ assert.Equal(t, expected.Size, actual.Size)
+ assert.Equal(t, expected.DownloadCount, actual.DownloadCount)
+ assertTimeEqual(t, expected.Created, actual.Created)
+ assertTimeEqual(t, expected.Updated, actual.Updated)
+ assert.Equal(t, expected.DownloadURL, actual.DownloadURL)
+}
+
+func assertReleaseAssetsEqual(t *testing.T, expected, actual []*base.ReleaseAsset) {
+ if assert.Len(t, actual, len(expected)) {
+ for i := range expected {
+ assertReleaseAssetEqual(t, expected[i], actual[i])
+ }
+ }
+}
+
+func assertReleaseEqual(t *testing.T, expected, actual *base.Release) {
+ assert.Equal(t, expected.TagName, actual.TagName)
+ assert.Equal(t, expected.TargetCommitish, actual.TargetCommitish)
+ assert.Equal(t, expected.Name, actual.Name)
+ assert.Equal(t, expected.Body, actual.Body)
+ assert.Equal(t, expected.Draft, actual.Draft)
+ assert.Equal(t, expected.Prerelease, actual.Prerelease)
+ assert.Equal(t, expected.PublisherID, actual.PublisherID)
+ assert.Equal(t, expected.PublisherName, actual.PublisherName)
+ assert.Equal(t, expected.PublisherEmail, actual.PublisherEmail)
+ assertReleaseAssetsEqual(t, expected.Assets, actual.Assets)
+ assertTimeEqual(t, expected.Created, actual.Created)
+ assertTimeEqual(t, expected.Published, actual.Published)
+}
+
+func assertReleasesEqual(t *testing.T, expected, actual []*base.Release) {
+ if assert.Len(t, actual, len(expected)) {
+ for i := range expected {
+ assertReleaseEqual(t, expected[i], actual[i])
+ }
+ }
+}
+
+func assertRepositoryEqual(t *testing.T, expected, actual *base.Repository) {
+ assert.Equal(t, expected.Name, actual.Name)
+ assert.Equal(t, expected.Owner, actual.Owner)
+ assert.Equal(t, expected.IsPrivate, actual.IsPrivate)
+ assert.Equal(t, expected.IsMirror, actual.IsMirror)
+ assert.Equal(t, expected.Description, actual.Description)
+ assert.Equal(t, expected.CloneURL, actual.CloneURL)
+ assert.Equal(t, expected.OriginalURL, actual.OriginalURL)
+ assert.Equal(t, expected.DefaultBranch, actual.DefaultBranch)
+}
+
+func assertReviewEqual(t *testing.T, expected, actual *base.Review) {
+ assert.Equal(t, expected.ID, actual.ID)
+ assert.Equal(t, expected.IssueIndex, actual.IssueIndex)
+ assert.Equal(t, expected.ReviewerID, actual.ReviewerID)
+ assert.Equal(t, expected.ReviewerName, actual.ReviewerName)
+ assert.Equal(t, expected.Official, actual.Official)
+ assert.Equal(t, expected.CommitID, actual.CommitID)
+ assert.Equal(t, expected.Content, actual.Content)
+ assertTimeEqual(t, expected.CreatedAt, actual.CreatedAt)
+ assert.Equal(t, expected.State, actual.State)
+ assertReviewCommentsEqual(t, expected.Comments, actual.Comments)
+}
+
+func assertReviewsEqual(t *testing.T, expected, actual []*base.Review) {
+ if assert.Len(t, actual, len(expected)) {
+ for i := range expected {
+ assertReviewEqual(t, expected[i], actual[i])
+ }
+ }
+}
+
+func assertReviewCommentEqual(t *testing.T, expected, actual *base.ReviewComment) {
+ assert.Equal(t, expected.ID, actual.ID)
+ assert.Equal(t, expected.InReplyTo, actual.InReplyTo)
+ assert.Equal(t, expected.Content, actual.Content)
+ assert.Equal(t, expected.TreePath, actual.TreePath)
+ assert.Equal(t, expected.DiffHunk, actual.DiffHunk)
+ assert.Equal(t, expected.Position, actual.Position)
+ assert.Equal(t, expected.Line, actual.Line)
+ assert.Equal(t, expected.CommitID, actual.CommitID)
+ assert.Equal(t, expected.PosterID, actual.PosterID)
+ assertReactionsEqual(t, expected.Reactions, actual.Reactions)
+ assertTimeEqual(t, expected.CreatedAt, actual.CreatedAt)
+ assertTimeEqual(t, expected.UpdatedAt, actual.UpdatedAt)
+}
+
+func assertReviewCommentsEqual(t *testing.T, expected, actual []*base.ReviewComment) {
+ if assert.Len(t, actual, len(expected)) {
+ for i := range expected {
+ assertReviewCommentEqual(t, expected[i], actual[i])
+ }
+ }
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "fmt"
+ "net"
+ "net/url"
+ "path/filepath"
+ "strings"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/matchlist"
+ base "code.gitea.io/gitea/modules/migration"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/util"
+)
+
+// MigrateOptions is equal to base.MigrateOptions
+type MigrateOptions = base.MigrateOptions
+
+var (
+ factories []base.DownloaderFactory
+
+ allowList *matchlist.Matchlist
+ blockList *matchlist.Matchlist
+)
+
+// RegisterDownloaderFactory registers a downloader factory
+func RegisterDownloaderFactory(factory base.DownloaderFactory) {
+ factories = append(factories, factory)
+}
+
+// IsMigrateURLAllowed checks if an URL is allowed to be migrated from
+func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
+ // Remote address can be HTTP/HTTPS/Git URL or local path.
+ u, err := url.Parse(remoteURL)
+ if err != nil {
+ return &models.ErrInvalidCloneAddr{IsURLError: true}
+ }
+
+ if u.Scheme == "file" || u.Scheme == "" {
+ if !doer.CanImportLocal() {
+ return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsPermissionDenied: true, LocalPath: true}
+ }
+ isAbs := filepath.IsAbs(u.Host + u.Path)
+ if !isAbs {
+ return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
+ }
+ isDir, err := util.IsDir(u.Host + u.Path)
+ if err != nil {
+ log.Error("Unable to check if %s is a directory: %v", u.Host+u.Path, err)
+ return err
+ }
+ if !isDir {
+ return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
+ }
+
+ return nil
+ }
+
+ if u.Scheme == "git" && u.Port() != "" && (strings.Contains(remoteURL, "%0d") || strings.Contains(remoteURL, "%0a")) {
+ return &models.ErrInvalidCloneAddr{Host: u.Host, IsURLError: true}
+ }
+
+ if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" {
+ return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true}
+ }
+
+ host := strings.ToLower(u.Host)
+ if len(setting.Migrations.AllowedDomains) > 0 {
+ if !allowList.Match(host) {
+ return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
+ }
+ } else {
+ if blockList.Match(host) {
+ return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
+ }
+ }
+
+ if !setting.Migrations.AllowLocalNetworks {
+ addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0])
+ if err != nil {
+ return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true}
+ }
+ for _, addr := range addrList {
+ if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() {
+ return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true}
+ }
+ }
+ }
+
+ return nil
+}
+
+// MigrateRepository migrate repository according MigrateOptions
+func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*models.Repository, error) {
+ err := IsMigrateURLAllowed(opts.CloneAddr, doer)
+ if err != nil {
+ return nil, err
+ }
+ if opts.LFS && len(opts.LFSEndpoint) > 0 {
+ err := IsMigrateURLAllowed(opts.LFSEndpoint, doer)
+ if err != nil {
+ return nil, err
+ }
+ }
+ downloader, err := newDownloader(ctx, ownerName, opts)
+ if err != nil {
+ return nil, err
+ }
+
+ var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
+ uploader.gitServiceType = opts.GitServiceType
+
+ if err := migrateRepository(downloader, uploader, opts, messenger); err != nil {
+ if err1 := uploader.Rollback(); err1 != nil {
+ log.Error("rollback failed: %v", err1)
+ }
+ if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
+ log.Error("create respotiry notice failed: ", err2)
+ }
+ return nil, err
+ }
+ return uploader.repo, nil
+}
+
+func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) {
+ var (
+ downloader base.Downloader
+ err error
+ )
+
+ for _, factory := range factories {
+ if factory.GitServiceType() == opts.GitServiceType {
+ downloader, err = factory.New(ctx, opts)
+ if err != nil {
+ return nil, err
+ }
+ break
+ }
+ }
+
+ if downloader == nil {
+ opts.Wiki = true
+ opts.Milestones = false
+ opts.Labels = false
+ opts.Releases = false
+ opts.Comments = false
+ opts.Issues = false
+ opts.PullRequests = false
+ downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
+ log.Trace("Will migrate from git: %s", opts.OriginalURL)
+ }
+
+ if setting.Migrations.MaxAttempts > 1 {
+ downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
+ }
+ return downloader, nil
+}
+
+// migrateRepository will download information and then upload it to Uploader, this is a simple
+// process for small repository. For a big repository, save all the data to disk
+// before upload is better
+func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
+ if messenger == nil {
+ messenger = base.NilMessenger
+ }
+
+ repo, err := downloader.GetRepoInfo()
+ if err != nil {
+ if !base.IsErrNotSupported(err) {
+ return err
+ }
+ log.Info("migrating repo infos is not supported, ignored")
+ }
+ repo.IsPrivate = opts.Private
+ repo.IsMirror = opts.Mirror
+ if opts.Description != "" {
+ repo.Description = opts.Description
+ }
+ if repo.CloneURL, err = downloader.FormatCloneURL(opts, repo.CloneURL); err != nil {
+ return err
+ }
+
+ log.Trace("migrating git data from %s", repo.CloneURL)
+ messenger("repo.migrate.migrating_git")
+ if err = uploader.CreateRepo(repo, opts); err != nil {
+ return err
+ }
+ defer uploader.Close()
+
+ log.Trace("migrating topics")
+ messenger("repo.migrate.migrating_topics")
+ topics, err := downloader.GetTopics()
+ if err != nil {
+ if !base.IsErrNotSupported(err) {
+ return err
+ }
+ log.Warn("migrating topics is not supported, ignored")
+ }
+ if len(topics) != 0 {
+ if err = uploader.CreateTopics(topics...); err != nil {
+ return err
+ }
+ }
+
+ if opts.Milestones {
+ log.Trace("migrating milestones")
+ messenger("repo.migrate.migrating_milestones")
+ milestones, err := downloader.GetMilestones()
+ if err != nil {
+ if !base.IsErrNotSupported(err) {
+ return err
+ }
+ log.Warn("migrating milestones is not supported, ignored")
+ }
+
+ msBatchSize := uploader.MaxBatchInsertSize("milestone")
+ for len(milestones) > 0 {
+ if len(milestones) < msBatchSize {
+ msBatchSize = len(milestones)
+ }
+
+ if err := uploader.CreateMilestones(milestones...); err != nil {
+ return err
+ }
+ milestones = milestones[msBatchSize:]
+ }
+ }
+
+ if opts.Labels {
+ log.Trace("migrating labels")
+ messenger("repo.migrate.migrating_labels")
+ labels, err := downloader.GetLabels()
+ if err != nil {
+ if !base.IsErrNotSupported(err) {
+ return err
+ }
+ log.Warn("migrating labels is not supported, ignored")
+ }
+
+ lbBatchSize := uploader.MaxBatchInsertSize("label")
+ for len(labels) > 0 {
+ if len(labels) < lbBatchSize {
+ lbBatchSize = len(labels)
+ }
+
+ if err := uploader.CreateLabels(labels...); err != nil {
+ return err
+ }
+ labels = labels[lbBatchSize:]
+ }
+ }
+
+ if opts.Releases {
+ log.Trace("migrating releases")
+ messenger("repo.migrate.migrating_releases")
+ releases, err := downloader.GetReleases()
+ if err != nil {
+ if !base.IsErrNotSupported(err) {
+ return err
+ }
+ log.Warn("migrating releases is not supported, ignored")
+ }
+
+ relBatchSize := uploader.MaxBatchInsertSize("release")
+ for len(releases) > 0 {
+ if len(releases) < relBatchSize {
+ relBatchSize = len(releases)
+ }
+
+ if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
+ return err
+ }
+ releases = releases[relBatchSize:]
+ }
+
+ // Once all releases (if any) are inserted, sync any remaining non-release tags
+ if err = uploader.SyncTags(); err != nil {
+ return err
+ }
+ }
+
+ var (
+ commentBatchSize = uploader.MaxBatchInsertSize("comment")
+ reviewBatchSize = uploader.MaxBatchInsertSize("review")
+ )
+
+ supportAllComments := downloader.SupportGetRepoComments()
+
+ if opts.Issues {
+ log.Trace("migrating issues and comments")
+ messenger("repo.migrate.migrating_issues")
+ var issueBatchSize = uploader.MaxBatchInsertSize("issue")
+
+ for i := 1; ; i++ {
+ issues, isEnd, err := downloader.GetIssues(i, issueBatchSize)
+ if err != nil {
+ if !base.IsErrNotSupported(err) {
+ return err
+ }
+ log.Warn("migrating issues is not supported, ignored")
+ break
+ }
+
+ if err := uploader.CreateIssues(issues...); err != nil {
+ return err
+ }
+
+ if opts.Comments && !supportAllComments {
+ var allComments = make([]*base.Comment, 0, commentBatchSize)
+ for _, issue := range issues {
+ log.Trace("migrating issue %d's comments", issue.Number)
+ comments, _, err := downloader.GetComments(base.GetCommentOptions{
+ Context: issue.Context,
+ })
+ if err != nil {
+ if !base.IsErrNotSupported(err) {
+ return err
+ }
+ log.Warn("migrating comments is not supported, ignored")
+ }
+
+ allComments = append(allComments, comments...)
+
+ if len(allComments) >= commentBatchSize {
+ if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
+ return err
+ }
+
+ allComments = allComments[commentBatchSize:]
+ }
+ }
+
+ if len(allComments) > 0 {
+ if err = uploader.CreateComments(allComments...); err != nil {
+ return err
+ }
+ }
+ }
+
+ if isEnd {
+ break
+ }
+ }
+ }
+
+ if opts.PullRequests {
+ log.Trace("migrating pull requests and comments")
+ messenger("repo.migrate.migrating_pulls")
+ var prBatchSize = uploader.MaxBatchInsertSize("pullrequest")
+ for i := 1; ; i++ {
+ prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)
+ if err != nil {
+ if !base.IsErrNotSupported(err) {
+ return err
+ }
+ log.Warn("migrating pull requests is not supported, ignored")
+ break
+ }
+
+ if err := uploader.CreatePullRequests(prs...); err != nil {
+ return err
+ }
+
+ if opts.Comments {
+ if !supportAllComments {
+ // plain comments
+ var allComments = make([]*base.Comment, 0, commentBatchSize)
+ for _, pr := range prs {
+ log.Trace("migrating pull request %d's comments", pr.Number)
+ comments, _, err := downloader.GetComments(base.GetCommentOptions{
+ Context: pr.Context,
+ })
+ if err != nil {
+ if !base.IsErrNotSupported(err) {
+ return err
+ }
+ log.Warn("migrating comments is not supported, ignored")
+ }
+
+ allComments = append(allComments, comments...)
+
+ if len(allComments) >= commentBatchSize {
+ if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
+ return err
+ }
+ allComments = allComments[commentBatchSize:]
+ }
+ }
+ if len(allComments) > 0 {
+ if err = uploader.CreateComments(allComments...); err != nil {
+ return err
+ }
+ }
+ }
+
+ // migrate reviews
+ var allReviews = make([]*base.Review, 0, reviewBatchSize)
+ for _, pr := range prs {
+ reviews, err := downloader.GetReviews(pr.Context)
+ if err != nil {
+ if !base.IsErrNotSupported(err) {
+ return err
+ }
+ log.Warn("migrating reviews is not supported, ignored")
+ break
+ }
+
+ allReviews = append(allReviews, reviews...)
+
+ if len(allReviews) >= reviewBatchSize {
+ if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
+ return err
+ }
+ allReviews = allReviews[reviewBatchSize:]
+ }
+ }
+ if len(allReviews) > 0 {
+ if err = uploader.CreateReviews(allReviews...); err != nil {
+ return err
+ }
+ }
+ }
+
+ if isEnd {
+ break
+ }
+ }
+ }
+
+ if opts.Comments && supportAllComments {
+ log.Trace("migrating comments")
+ for i := 1; ; i++ {
+ comments, isEnd, err := downloader.GetComments(base.GetCommentOptions{
+ Page: i,
+ PageSize: commentBatchSize,
+ })
+ if err != nil {
+ return err
+ }
+
+ if err := uploader.CreateComments(comments...); err != nil {
+ return err
+ }
+
+ if isEnd {
+ break
+ }
+ }
+ }
+
+ return uploader.Finish()
+}
+
+// Init migrations service
+func Init() error {
+ var err error
+ allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...)
+ if err != nil {
+ return fmt.Errorf("init migration allowList domains failed: %v", err)
+ }
+
+ blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...)
+ if err != nil {
+ return fmt.Errorf("init migration blockList domains failed: %v", err)
+ }
+
+ return nil
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "path/filepath"
+ "testing"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/models/unittest"
+ "code.gitea.io/gitea/modules/setting"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMigrateWhiteBlocklist(t *testing.T) {
+ assert.NoError(t, unittest.PrepareTestDatabase())
+
+ adminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user1"}).(*models.User)
+ nonAdminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user2"}).(*models.User)
+
+ setting.Migrations.AllowedDomains = []string{"github.com"}
+ assert.NoError(t, Init())
+
+ err := IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
+ assert.Error(t, err)
+
+ err = IsMigrateURLAllowed("https://github.com/go-gitea/gitea.git", nonAdminUser)
+ assert.NoError(t, err)
+
+ err = IsMigrateURLAllowed("https://gITHUb.com/go-gitea/gitea.git", nonAdminUser)
+ assert.NoError(t, err)
+
+ setting.Migrations.AllowedDomains = []string{}
+ setting.Migrations.BlockedDomains = []string{"github.com"}
+ assert.NoError(t, Init())
+
+ err = IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
+ assert.NoError(t, err)
+
+ err = IsMigrateURLAllowed("https://github.com/go-gitea/gitea.git", nonAdminUser)
+ assert.Error(t, err)
+
+ err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
+ assert.Error(t, err)
+
+ setting.Migrations.AllowLocalNetworks = true
+ err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
+ assert.NoError(t, err)
+
+ old := setting.ImportLocalPaths
+ setting.ImportLocalPaths = false
+
+ err = IsMigrateURLAllowed("/home/foo/bar/goo", adminUser)
+ assert.Error(t, err)
+
+ setting.ImportLocalPaths = true
+ abs, err := filepath.Abs(".")
+ assert.NoError(t, err)
+
+ err = IsMigrateURLAllowed(abs, adminUser)
+ assert.NoError(t, err)
+
+ err = IsMigrateURLAllowed(abs, nonAdminUser)
+ assert.Error(t, err)
+
+ nonAdminUser.AllowImportLocal = true
+ err = IsMigrateURLAllowed(abs, nonAdminUser)
+ assert.NoError(t, err)
+
+ setting.ImportLocalPaths = old
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+
+ "code.gitea.io/gitea/modules/json"
+ "code.gitea.io/gitea/modules/log"
+ base "code.gitea.io/gitea/modules/migration"
+ "code.gitea.io/gitea/modules/structs"
+)
+
+var (
+ _ base.Downloader = &OneDevDownloader{}
+ _ base.DownloaderFactory = &OneDevDownloaderFactory{}
+)
+
+func init() {
+ RegisterDownloaderFactory(&OneDevDownloaderFactory{})
+}
+
+// OneDevDownloaderFactory defines a downloader factory
+type OneDevDownloaderFactory struct {
+}
+
+// New returns a downloader related to this factory according MigrateOptions
+func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
+ u, err := url.Parse(opts.CloneAddr)
+ if err != nil {
+ return nil, err
+ }
+
+ repoName := ""
+
+ fields := strings.Split(strings.Trim(u.Path, "/"), "/")
+ if len(fields) == 2 && fields[0] == "projects" {
+ repoName = fields[1]
+ } else if len(fields) == 1 {
+ repoName = fields[0]
+ } else {
+ return nil, fmt.Errorf("invalid path: %s", u.Path)
+ }
+
+ u.Path = ""
+ u.Fragment = ""
+
+ log.Trace("Create onedev downloader. BaseURL: %v RepoName: %s", u, repoName)
+
+ return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoName), nil
+}
+
+// GitServiceType returns the type of git service
+func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType {
+ return structs.OneDevService
+}
+
+type onedevUser struct {
+ ID int64 `json:"id"`
+ Name string `json:"name"`
+ Email string `json:"email"`
+}
+
+// OneDevDownloader implements a Downloader interface to get repository informations
+// from OneDev
+type OneDevDownloader struct {
+ base.NullDownloader
+ ctx context.Context
+ client *http.Client
+ baseURL *url.URL
+ repoName string
+ repoID int64
+ maxIssueIndex int64
+ userMap map[int64]*onedevUser
+ milestoneMap map[int64]string
+}
+
+// SetContext set context
+func (d *OneDevDownloader) SetContext(ctx context.Context) {
+ d.ctx = ctx
+}
+
+// NewOneDevDownloader creates a new downloader
+func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader {
+ var downloader = &OneDevDownloader{
+ ctx: ctx,
+ baseURL: baseURL,
+ repoName: repoName,
+ client: &http.Client{
+ Transport: &http.Transport{
+ Proxy: func(req *http.Request) (*url.URL, error) {
+ if len(username) > 0 && len(password) > 0 {
+ req.SetBasicAuth(username, password)
+ }
+ return nil, nil
+ },
+ },
+ },
+ userMap: make(map[int64]*onedevUser),
+ milestoneMap: make(map[int64]string),
+ }
+
+ return downloader
+}
+
+func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error {
+ u, err := d.baseURL.Parse(endpoint)
+ if err != nil {
+ return err
+ }
+
+ if parameter != nil {
+ query := u.Query()
+ for k, v := range parameter {
+ query.Set(k, v)
+ }
+ u.RawQuery = query.Encode()
+ }
+
+ req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil)
+ if err != nil {
+ return err
+ }
+
+ resp, err := d.client.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+
+ decoder := json.NewDecoder(resp.Body)
+ return decoder.Decode(&result)
+}
+
+// GetRepoInfo returns repository information
+func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) {
+ info := make([]struct {
+ ID int64 `json:"id"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+ }, 0, 1)
+
+ err := d.callAPI(
+ "/api/projects",
+ map[string]string{
+ "query": `"Name" is "` + d.repoName + `"`,
+ "offset": "0",
+ "count": "1",
+ },
+ &info,
+ )
+ if err != nil {
+ return nil, err
+ }
+ if len(info) != 1 {
+ return nil, fmt.Errorf("Project %s not found", d.repoName)
+ }
+
+ d.repoID = info[0].ID
+
+ cloneURL, err := d.baseURL.Parse(info[0].Name)
+ if err != nil {
+ return nil, err
+ }
+ originalURL, err := d.baseURL.Parse("/projects/" + info[0].Name)
+ if err != nil {
+ return nil, err
+ }
+
+ return &base.Repository{
+ Name: info[0].Name,
+ Description: info[0].Description,
+ CloneURL: cloneURL.String(),
+ OriginalURL: originalURL.String(),
+ }, nil
+}
+
+// GetMilestones returns milestones
+func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) {
+ rawMilestones := make([]struct {
+ ID int64 `json:"id"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+ DueDate *time.Time `json:"dueDate"`
+ Closed bool `json:"closed"`
+ }, 0, 100)
+
+ endpoint := fmt.Sprintf("/api/projects/%d/milestones", d.repoID)
+
+ var milestones = make([]*base.Milestone, 0, 100)
+ offset := 0
+ for {
+ err := d.callAPI(
+ endpoint,
+ map[string]string{
+ "offset": strconv.Itoa(offset),
+ "count": "100",
+ },
+ &rawMilestones,
+ )
+ if err != nil {
+ return nil, err
+ }
+ if len(rawMilestones) == 0 {
+ break
+ }
+ offset += 100
+
+ for _, milestone := range rawMilestones {
+ d.milestoneMap[milestone.ID] = milestone.Name
+ closed := milestone.DueDate
+ if !milestone.Closed {
+ closed = nil
+ }
+
+ milestones = append(milestones, &base.Milestone{
+ Title: milestone.Name,
+ Description: milestone.Description,
+ Deadline: milestone.DueDate,
+ Closed: closed,
+ })
+ }
+ }
+ return milestones, nil
+}
+
+// GetLabels returns labels
+func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) {
+ return []*base.Label{
+ {
+ Name: "Bug",
+ Color: "f64e60",
+ },
+ {
+ Name: "Build Failure",
+ Color: "f64e60",
+ },
+ {
+ Name: "Discussion",
+ Color: "8950fc",
+ },
+ {
+ Name: "Improvement",
+ Color: "1bc5bd",
+ },
+ {
+ Name: "New Feature",
+ Color: "1bc5bd",
+ },
+ {
+ Name: "Support Request",
+ Color: "8950fc",
+ },
+ }, nil
+}
+
+type onedevIssueContext struct {
+ foreignID int64
+ localID int64
+ IsPullRequest bool
+}
+
+func (c onedevIssueContext) LocalID() int64 {
+ return c.localID
+}
+
+func (c onedevIssueContext) ForeignID() int64 {
+ return c.foreignID
+}
+
+// GetIssues returns issues
+func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
+ rawIssues := make([]struct {
+ ID int64 `json:"id"`
+ Number int64 `json:"number"`
+ State string `json:"state"`
+ Title string `json:"title"`
+ Description string `json:"description"`
+ MilestoneID int64 `json:"milestoneId"`
+ SubmitterID int64 `json:"submitterId"`
+ SubmitDate time.Time `json:"submitDate"`
+ }, 0, perPage)
+
+ err := d.callAPI(
+ "/api/issues",
+ map[string]string{
+ "query": `"Project" is "` + d.repoName + `"`,
+ "offset": strconv.Itoa((page - 1) * perPage),
+ "count": strconv.Itoa(perPage),
+ },
+ &rawIssues,
+ )
+ if err != nil {
+ return nil, false, err
+ }
+
+ issues := make([]*base.Issue, 0, len(rawIssues))
+ for _, issue := range rawIssues {
+ fields := make([]struct {
+ Name string `json:"name"`
+ Value string `json:"value"`
+ }, 0, 10)
+ err := d.callAPI(
+ fmt.Sprintf("/api/issues/%d/fields", issue.ID),
+ nil,
+ &fields,
+ )
+ if err != nil {
+ return nil, false, err
+ }
+
+ var label *base.Label
+ for _, field := range fields {
+ if field.Name == "Type" {
+ label = &base.Label{Name: field.Value}
+ break
+ }
+ }
+
+ state := strings.ToLower(issue.State)
+ if state == "released" {
+ state = "closed"
+ }
+ poster := d.tryGetUser(issue.SubmitterID)
+ issues = append(issues, &base.Issue{
+ Title: issue.Title,
+ Number: issue.Number,
+ PosterName: poster.Name,
+ PosterEmail: poster.Email,
+ Content: issue.Description,
+ Milestone: d.milestoneMap[issue.MilestoneID],
+ State: state,
+ Created: issue.SubmitDate,
+ Updated: issue.SubmitDate,
+ Labels: []*base.Label{label},
+ Context: onedevIssueContext{
+ foreignID: issue.ID,
+ localID: issue.Number,
+ IsPullRequest: false,
+ },
+ })
+
+ if d.maxIssueIndex < issue.Number {
+ d.maxIssueIndex = issue.Number
+ }
+ }
+
+ return issues, len(issues) == 0, nil
+}
+
+// GetComments returns comments
+func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
+ context, ok := opts.Context.(onedevIssueContext)
+ if !ok {
+ return nil, false, fmt.Errorf("unexpected comment context: %+v", opts.Context)
+ }
+
+ rawComments := make([]struct {
+ Date time.Time `json:"date"`
+ UserID int64 `json:"userId"`
+ Content string `json:"content"`
+ }, 0, 100)
+
+ var endpoint string
+ if context.IsPullRequest {
+ endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", context.ForeignID())
+ } else {
+ endpoint = fmt.Sprintf("/api/issues/%d/comments", context.ForeignID())
+ }
+
+ err := d.callAPI(
+ endpoint,
+ nil,
+ &rawComments,
+ )
+ if err != nil {
+ return nil, false, err
+ }
+
+ rawChanges := make([]struct {
+ Date time.Time `json:"date"`
+ UserID int64 `json:"userId"`
+ Data map[string]interface{} `json:"data"`
+ }, 0, 100)
+
+ if context.IsPullRequest {
+ endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", context.ForeignID())
+ } else {
+ endpoint = fmt.Sprintf("/api/issues/%d/changes", context.ForeignID())
+ }
+
+ err = d.callAPI(
+ endpoint,
+ nil,
+ &rawChanges,
+ )
+ if err != nil {
+ return nil, false, err
+ }
+
+ comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges))
+ for _, comment := range rawComments {
+ if len(comment.Content) == 0 {
+ continue
+ }
+ poster := d.tryGetUser(comment.UserID)
+ comments = append(comments, &base.Comment{
+ IssueIndex: context.LocalID(),
+ PosterID: poster.ID,
+ PosterName: poster.Name,
+ PosterEmail: poster.Email,
+ Content: comment.Content,
+ Created: comment.Date,
+ Updated: comment.Date,
+ })
+ }
+ for _, change := range rawChanges {
+ contentV, ok := change.Data["content"]
+ if !ok {
+ contentV, ok = change.Data["comment"]
+ if !ok {
+ continue
+ }
+ }
+ content, ok := contentV.(string)
+ if !ok || len(content) == 0 {
+ continue
+ }
+
+ poster := d.tryGetUser(change.UserID)
+ comments = append(comments, &base.Comment{
+ IssueIndex: context.LocalID(),
+ PosterID: poster.ID,
+ PosterName: poster.Name,
+ PosterEmail: poster.Email,
+ Content: content,
+ Created: change.Date,
+ Updated: change.Date,
+ })
+ }
+
+ return comments, true, nil
+}
+
+// GetPullRequests returns pull requests
+func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
+ rawPullRequests := make([]struct {
+ ID int64 `json:"id"`
+ Number int64 `json:"number"`
+ Title string `json:"title"`
+ SubmitterID int64 `json:"submitterId"`
+ SubmitDate time.Time `json:"submitDate"`
+ Description string `json:"description"`
+ TargetBranch string `json:"targetBranch"`
+ SourceBranch string `json:"sourceBranch"`
+ BaseCommitHash string `json:"baseCommitHash"`
+ CloseInfo *struct {
+ Date *time.Time `json:"date"`
+ Status string `json:"status"`
+ }
+ }, 0, perPage)
+
+ err := d.callAPI(
+ "/api/pull-requests",
+ map[string]string{
+ "query": `"Target Project" is "` + d.repoName + `"`,
+ "offset": strconv.Itoa((page - 1) * perPage),
+ "count": strconv.Itoa(perPage),
+ },
+ &rawPullRequests,
+ )
+ if err != nil {
+ return nil, false, err
+ }
+
+ pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests))
+ for _, pr := range rawPullRequests {
+ var mergePreview struct {
+ TargetHeadCommitHash string `json:"targetHeadCommitHash"`
+ HeadCommitHash string `json:"headCommitHash"`
+ MergeStrategy string `json:"mergeStrategy"`
+ MergeCommitHash string `json:"mergeCommitHash"`
+ }
+ err := d.callAPI(
+ fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID),
+ nil,
+ &mergePreview,
+ )
+ if err != nil {
+ return nil, false, err
+ }
+
+ state := "open"
+ merged := false
+ var closeTime *time.Time
+ var mergedTime *time.Time
+ if pr.CloseInfo != nil {
+ state = "closed"
+ closeTime = pr.CloseInfo.Date
+ if pr.CloseInfo.Status == "MERGED" { // "DISCARDED"
+ merged = true
+ mergedTime = pr.CloseInfo.Date
+ }
+ }
+ poster := d.tryGetUser(pr.SubmitterID)
+
+ number := pr.Number + d.maxIssueIndex
+ pullRequests = append(pullRequests, &base.PullRequest{
+ Title: pr.Title,
+ Number: number,
+ PosterName: poster.Name,
+ PosterID: poster.ID,
+ Content: pr.Description,
+ State: state,
+ Created: pr.SubmitDate,
+ Updated: pr.SubmitDate,
+ Closed: closeTime,
+ Merged: merged,
+ MergedTime: mergedTime,
+ Head: base.PullRequestBranch{
+ Ref: pr.SourceBranch,
+ SHA: mergePreview.HeadCommitHash,
+ RepoName: d.repoName,
+ },
+ Base: base.PullRequestBranch{
+ Ref: pr.TargetBranch,
+ SHA: mergePreview.TargetHeadCommitHash,
+ RepoName: d.repoName,
+ },
+ Context: onedevIssueContext{
+ foreignID: pr.ID,
+ localID: number,
+ IsPullRequest: true,
+ },
+ })
+ }
+
+ return pullRequests, len(pullRequests) == 0, nil
+}
+
+// GetReviews returns pull requests reviews
+func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
+ rawReviews := make([]struct {
+ ID int64 `json:"id"`
+ UserID int64 `json:"userId"`
+ Result *struct {
+ Commit string `json:"commit"`
+ Approved bool `json:"approved"`
+ Comment string `json:"comment"`
+ }
+ }, 0, 100)
+
+ err := d.callAPI(
+ fmt.Sprintf("/api/pull-requests/%d/reviews", context.ForeignID()),
+ nil,
+ &rawReviews,
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ var reviews = make([]*base.Review, 0, len(rawReviews))
+ for _, review := range rawReviews {
+ state := base.ReviewStatePending
+ content := ""
+ if review.Result != nil {
+ if len(review.Result.Comment) > 0 {
+ state = base.ReviewStateCommented
+ content = review.Result.Comment
+ }
+ if review.Result.Approved {
+ state = base.ReviewStateApproved
+ }
+ }
+
+ poster := d.tryGetUser(review.UserID)
+ reviews = append(reviews, &base.Review{
+ IssueIndex: context.LocalID(),
+ ReviewerID: poster.ID,
+ ReviewerName: poster.Name,
+ Content: content,
+ State: state,
+ })
+ }
+
+ return reviews, nil
+}
+
+// GetTopics return repository topics
+func (d *OneDevDownloader) GetTopics() ([]string, error) {
+ return []string{}, nil
+}
+
+func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser {
+ user, ok := d.userMap[userID]
+ if !ok {
+ err := d.callAPI(
+ fmt.Sprintf("/api/users/%d", userID),
+ nil,
+ &user,
+ )
+ if err != nil {
+ user = &onedevUser{
+ Name: fmt.Sprintf("User %d", userID),
+ }
+ }
+ d.userMap[userID] = user
+ }
+
+ return user
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/url"
+ "testing"
+ "time"
+
+ base "code.gitea.io/gitea/modules/migration"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestOneDevDownloadRepo(t *testing.T) {
+ resp, err := http.Get("https://code.onedev.io/projects/go-gitea-test_repo")
+ if err != nil || resp.StatusCode != 200 {
+ t.Skipf("Can't access test repo, skipping %s", t.Name())
+ }
+
+ u, _ := url.Parse("https://code.onedev.io")
+ downloader := NewOneDevDownloader(context.Background(), u, "", "", "go-gitea-test_repo")
+ if err != nil {
+ t.Fatal(fmt.Sprintf("NewOneDevDownloader is nil: %v", err))
+ }
+ repo, err := downloader.GetRepoInfo()
+ assert.NoError(t, err)
+ assertRepositoryEqual(t, &base.Repository{
+ Name: "go-gitea-test_repo",
+ Owner: "",
+ Description: "Test repository for testing migration from OneDev to gitea",
+ CloneURL: "https://code.onedev.io/go-gitea-test_repo",
+ OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo",
+ }, repo)
+
+ milestones, err := downloader.GetMilestones()
+ assert.NoError(t, err)
+ deadline := time.Unix(1620086400, 0)
+ assertMilestonesEqual(t, []*base.Milestone{
+ {
+ Title: "1.0.0",
+ Deadline: &deadline,
+ Closed: &deadline,
+ },
+ {
+ Title: "1.1.0",
+ Description: "next things?",
+ },
+ }, milestones)
+
+ labels, err := downloader.GetLabels()
+ assert.NoError(t, err)
+ assert.Len(t, labels, 6)
+
+ issues, isEnd, err := downloader.GetIssues(1, 2)
+ assert.NoError(t, err)
+ assert.False(t, isEnd)
+ assertIssuesEqual(t, []*base.Issue{
+ {
+ Number: 4,
+ Title: "Hi there",
+ Content: "an issue not assigned to a milestone",
+ PosterName: "User 336",
+ State: "open",
+ Created: time.Unix(1628549776, 734000000),
+ Updated: time.Unix(1628549776, 734000000),
+ Labels: []*base.Label{
+ {
+ Name: "Improvement",
+ },
+ },
+ Context: onedevIssueContext{
+ foreignID: 398,
+ localID: 4,
+ IsPullRequest: false,
+ },
+ },
+ {
+ Number: 3,
+ Title: "Add an awesome feature",
+ Content: "just another issue to test against",
+ PosterName: "User 336",
+ State: "open",
+ Milestone: "1.1.0",
+ Created: time.Unix(1628549749, 878000000),
+ Updated: time.Unix(1628549749, 878000000),
+ Labels: []*base.Label{
+ {
+ Name: "New Feature",
+ },
+ },
+ Context: onedevIssueContext{
+ foreignID: 397,
+ localID: 3,
+ IsPullRequest: false,
+ },
+ },
+ }, issues)
+
+ comments, _, err := downloader.GetComments(base.GetCommentOptions{
+ Context: onedevIssueContext{
+ foreignID: 398,
+ localID: 4,
+ IsPullRequest: false,
+ },
+ })
+ assert.NoError(t, err)
+ assertCommentsEqual(t, []*base.Comment{
+ {
+ IssueIndex: 4,
+ PosterName: "User 336",
+ Created: time.Unix(1628549791, 128000000),
+ Updated: time.Unix(1628549791, 128000000),
+ Content: "it has a comment\n\nEDIT: that got edited",
+ },
+ }, comments)
+
+ prs, _, err := downloader.GetPullRequests(1, 1)
+ assert.NoError(t, err)
+ assertPullRequestsEqual(t, []*base.PullRequest{
+ {
+ Number: 5,
+ Title: "Pull to add a new file",
+ Content: "just do some git stuff",
+ PosterName: "User 336",
+ State: "open",
+ Created: time.Unix(1628550076, 25000000),
+ Updated: time.Unix(1628550076, 25000000),
+ Head: base.PullRequestBranch{
+ Ref: "branch-for-a-pull",
+ SHA: "343deffe3526b9bc84e873743ff7f6e6d8b827c0",
+ RepoName: "go-gitea-test_repo",
+ },
+ Base: base.PullRequestBranch{
+ Ref: "master",
+ SHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
+ RepoName: "go-gitea-test_repo",
+ },
+ Context: onedevIssueContext{
+ foreignID: 186,
+ localID: 5,
+ IsPullRequest: true,
+ },
+ },
+ }, prs)
+
+ rvs, err := downloader.GetReviews(onedevIssueContext{
+ foreignID: 186,
+ localID: 5,
+ })
+ assert.NoError(t, err)
+ assertReviewsEqual(t, []*base.Review{
+ {
+ IssueIndex: 5,
+ ReviewerName: "User 317",
+ State: "PENDING",
+ },
+ }, rvs)
+}
--- /dev/null
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+
+ base "code.gitea.io/gitea/modules/migration"
+
+ "gopkg.in/yaml.v2"
+)
+
+// RepositoryRestorer implements an Downloader from the local directory
+type RepositoryRestorer struct {
+ base.NullDownloader
+ ctx context.Context
+ baseDir string
+ repoOwner string
+ repoName string
+}
+
+// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder
+func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) (*RepositoryRestorer, error) {
+ baseDir, err := filepath.Abs(baseDir)
+ if err != nil {
+ return nil, err
+ }
+ return &RepositoryRestorer{
+ ctx: ctx,
+ baseDir: baseDir,
+ repoOwner: owner,
+ repoName: repoName,
+ }, nil
+}
+
+func (r *RepositoryRestorer) commentDir() string {
+ return filepath.Join(r.baseDir, "comments")
+}
+
+func (r *RepositoryRestorer) reviewDir() string {
+ return filepath.Join(r.baseDir, "reviews")
+}
+
+// SetContext set context
+func (r *RepositoryRestorer) SetContext(ctx context.Context) {
+ r.ctx = ctx
+}
+
+func (r *RepositoryRestorer) getRepoOptions() (map[string]string, error) {
+ p := filepath.Join(r.baseDir, "repo.yml")
+ bs, err := os.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ var opts = make(map[string]string)
+ err = yaml.Unmarshal(bs, &opts)
+ if err != nil {
+ return nil, err
+ }
+ return opts, nil
+}
+
+// GetRepoInfo returns a repository information
+func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) {
+ opts, err := r.getRepoOptions()
+ if err != nil {
+ return nil, err
+ }
+
+ isPrivate, _ := strconv.ParseBool(opts["is_private"])
+
+ return &base.Repository{
+ Owner: r.repoOwner,
+ Name: r.repoName,
+ IsPrivate: isPrivate,
+ Description: opts["description"],
+ OriginalURL: opts["original_url"],
+ CloneURL: filepath.Join(r.baseDir, "git"),
+ DefaultBranch: opts["default_branch"],
+ }, nil
+}
+
+// GetTopics return github topics
+func (r *RepositoryRestorer) GetTopics() ([]string, error) {
+ p := filepath.Join(r.baseDir, "topic.yml")
+
+ var topics = struct {
+ Topics []string `yaml:"topics"`
+ }{}
+
+ bs, err := os.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &topics)
+ if err != nil {
+ return nil, err
+ }
+ return topics.Topics, nil
+}
+
+// GetMilestones returns milestones
+func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) {
+ var milestones = make([]*base.Milestone, 0, 10)
+ p := filepath.Join(r.baseDir, "milestone.yml")
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ bs, err := os.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &milestones)
+ if err != nil {
+ return nil, err
+ }
+ return milestones, nil
+}
+
+// GetReleases returns releases
+func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) {
+ var releases = make([]*base.Release, 0, 10)
+ p := filepath.Join(r.baseDir, "release.yml")
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ bs, err := os.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &releases)
+ if err != nil {
+ return nil, err
+ }
+ for _, rel := range releases {
+ for _, asset := range rel.Assets {
+ if asset.DownloadURL != nil {
+ *asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL)
+ }
+ }
+ }
+ return releases, nil
+}
+
+// GetLabels returns labels
+func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) {
+ var labels = make([]*base.Label, 0, 10)
+ p := filepath.Join(r.baseDir, "label.yml")
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ bs, err := os.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &labels)
+ if err != nil {
+ return nil, err
+ }
+ return labels, nil
+}
+
+// GetIssues returns issues according start and limit
+func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
+ var issues = make([]*base.Issue, 0, 10)
+ p := filepath.Join(r.baseDir, "issue.yml")
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, true, nil
+ }
+ return nil, false, err
+ }
+
+ bs, err := os.ReadFile(p)
+ if err != nil {
+ return nil, false, err
+ }
+
+ err = yaml.Unmarshal(bs, &issues)
+ if err != nil {
+ return nil, false, err
+ }
+ for _, issue := range issues {
+ issue.Context = base.BasicIssueContext(issue.Number)
+ }
+ return issues, true, nil
+}
+
+// GetComments returns comments according issueNumber
+func (r *RepositoryRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
+ var comments = make([]*base.Comment, 0, 10)
+ p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.Context.ForeignID()))
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, false, nil
+ }
+ return nil, false, err
+ }
+
+ bs, err := os.ReadFile(p)
+ if err != nil {
+ return nil, false, err
+ }
+
+ err = yaml.Unmarshal(bs, &comments)
+ if err != nil {
+ return nil, false, err
+ }
+ return comments, false, nil
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
+ var pulls = make([]*base.PullRequest, 0, 10)
+ p := filepath.Join(r.baseDir, "pull_request.yml")
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, true, nil
+ }
+ return nil, false, err
+ }
+
+ bs, err := os.ReadFile(p)
+ if err != nil {
+ return nil, false, err
+ }
+
+ err = yaml.Unmarshal(bs, &pulls)
+ if err != nil {
+ return nil, false, err
+ }
+ for _, pr := range pulls {
+ pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL)
+ pr.Context = base.BasicIssueContext(pr.Number)
+ }
+ return pulls, true, nil
+}
+
+// GetReviews returns pull requests review
+func (r *RepositoryRestorer) GetReviews(context base.IssueContext) ([]*base.Review, error) {
+ var reviews = make([]*base.Review, 0, 10)
+ p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", context.ForeignID()))
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ bs, err := os.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &reviews)
+ if err != nil {
+ return nil, err
+ }
+ return reviews, nil
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/models/db"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/structs"
+)
+
+// UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID
+func UpdateMigrationPosterID(ctx context.Context) error {
+ for _, gitService := range structs.SupportedFullGitService {
+ select {
+ case <-ctx.Done():
+ log.Warn("UpdateMigrationPosterID aborted before %s", gitService.Name())
+ return db.ErrCancelledf("during UpdateMigrationPosterID before %s", gitService.Name())
+ default:
+ }
+ if err := updateMigrationPosterIDByGitService(ctx, gitService); err != nil {
+ log.Error("updateMigrationPosterIDByGitService failed: %v", err)
+ }
+ }
+ return nil
+}
+
+func updateMigrationPosterIDByGitService(ctx context.Context, tp structs.GitServiceType) error {
+ provider := tp.Name()
+ if len(provider) == 0 {
+ return nil
+ }
+
+ const batchSize = 100
+ var start int
+ for {
+ select {
+ case <-ctx.Done():
+ log.Warn("UpdateMigrationPosterIDByGitService(%s) cancelled", tp.Name())
+ return nil
+ default:
+ }
+
+ users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{
+ Provider: provider,
+ Start: start,
+ Limit: batchSize,
+ })
+ if err != nil {
+ return err
+ }
+
+ for _, user := range users {
+ select {
+ case <-ctx.Done():
+ log.Warn("UpdateMigrationPosterIDByGitService(%s) cancelled", tp.Name())
+ return nil
+ default:
+ }
+ externalUserID := user.ExternalID
+ if err := models.UpdateMigrationsByType(tp, externalUserID, user.UserID); err != nil {
+ log.Error("UpdateMigrationsByType type %s external user id %v to local user id %v failed: %v", tp.Name(), user.ExternalID, user.UserID, err)
+ }
+ }
+
+ if len(users) < batchSize {
+ break
+ }
+ start += len(users)
+ }
+ return nil
+}
projects / gitea.git / commitdiff