diff options
author | Lunny Xiao <xiaolunwen@gmail.com> | 2021-11-16 23:25:33 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-11-16 23:25:33 +0800 |
commit | 7e1ae380975df0afab3fdc04c7a926181e5daba9 (patch) | |
tree | a6fa4eb2d15b88fc4ff953d748ee3937ff10446b /modules/migration | |
parent | 48ccd325a1b81a58ac6d1d5d94fc4e90974599ea (diff) | |
download | gitea-7e1ae380975df0afab3fdc04c7a926181e5daba9.tar.gz gitea-7e1ae380975df0afab3fdc04c7a926181e5daba9.zip |
Move migrations into services and base into modules/migration (#17663)
* Move migrtions into services and base into modules/migration
* Fix imports
* Fix lint
Diffstat (limited to 'modules/migration')
-rw-r--r-- | modules/migration/comment.go | 20 | ||||
-rw-r--r-- | modules/migration/downloader.go | 41 | ||||
-rw-r--r-- | modules/migration/error.go | 26 | ||||
-rw-r--r-- | modules/migration/issue.go | 48 | ||||
-rw-r--r-- | modules/migration/label.go | 13 | ||||
-rw-r--r-- | modules/migration/messenger.go | 11 | ||||
-rw-r--r-- | modules/migration/milestone.go | 19 | ||||
-rw-r--r-- | modules/migration/null_downloader.go | 87 | ||||
-rw-r--r-- | modules/migration/options.go | 42 | ||||
-rw-r--r-- | modules/migration/pullrequest.go | 61 | ||||
-rw-r--r-- | modules/migration/reaction.go | 12 | ||||
-rw-r--r-- | modules/migration/release.go | 40 | ||||
-rw-r--r-- | modules/migration/repo.go | 18 | ||||
-rw-r--r-- | modules/migration/retry_downloader.go | 197 | ||||
-rw-r--r-- | modules/migration/review.go | 45 | ||||
-rw-r--r-- | modules/migration/uploader.go | 24 |
16 files changed, 704 insertions, 0 deletions
diff --git a/modules/migration/comment.go b/modules/migration/comment.go new file mode 100644 index 0000000000..234fea3e82 --- /dev/null +++ b/modules/migration/comment.go @@ -0,0 +1,20 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import "time" + +// Comment is a standard comment information +type Comment struct { + IssueIndex int64 `yaml:"issue_index"` + PosterID int64 `yaml:"poster_id"` + PosterName string `yaml:"poster_name"` + PosterEmail string `yaml:"poster_email"` + Created time.Time + Updated time.Time + Content string + Reactions []*Reaction +} diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go new file mode 100644 index 0000000000..90e149fb1a --- /dev/null +++ b/modules/migration/downloader.go @@ -0,0 +1,41 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import ( + "context" + + "code.gitea.io/gitea/modules/structs" +) + +// GetCommentOptions represents an options for get comment +type GetCommentOptions struct { + Context IssueContext + Page int + PageSize int +} + +// Downloader downloads the site repo information +type Downloader interface { + SetContext(context.Context) + GetRepoInfo() (*Repository, error) + GetTopics() ([]string, error) + GetMilestones() ([]*Milestone, error) + GetReleases() ([]*Release, error) + GetLabels() ([]*Label, error) + GetIssues(page, perPage int) ([]*Issue, bool, error) + GetComments(opts GetCommentOptions) ([]*Comment, bool, error) + SupportGetRepoComments() bool + GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) + GetReviews(pullRequestContext IssueContext) ([]*Review, error) + FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) +} + +// DownloaderFactory defines an interface to match a downloader implementation and create a downloader +type DownloaderFactory interface { + New(ctx context.Context, opts MigrateOptions) (Downloader, error) + GitServiceType() structs.GitServiceType +} diff --git a/modules/migration/error.go b/modules/migration/error.go new file mode 100644 index 0000000000..b2608aa09f --- /dev/null +++ b/modules/migration/error.go @@ -0,0 +1,26 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import "fmt" + +// ErrNotSupported represents status if a downloader do not supported something. +type ErrNotSupported struct { + Entity string +} + +// IsErrNotSupported checks if an error is an ErrNotSupported +func IsErrNotSupported(err error) bool { + _, ok := err.(ErrNotSupported) + return ok +} + +// Error return error message +func (err ErrNotSupported) Error() string { + if len(err.Entity) != 0 { + return fmt.Sprintf("'%s' not supported", err.Entity) + } + return "not supported" +} diff --git a/modules/migration/issue.go b/modules/migration/issue.go new file mode 100644 index 0000000000..26812633f9 --- /dev/null +++ b/modules/migration/issue.go @@ -0,0 +1,48 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import "time" + +// IssueContext is used to map between local and foreign issue/PR ids. +type IssueContext interface { + LocalID() int64 + ForeignID() int64 +} + +// BasicIssueContext is a 1:1 mapping between local and foreign ids. +type BasicIssueContext int64 + +// LocalID gets the local id. +func (c BasicIssueContext) LocalID() int64 { + return int64(c) +} + +// ForeignID gets the foreign id. +func (c BasicIssueContext) ForeignID() int64 { + return int64(c) +} + +// Issue is a standard issue information +type Issue struct { + Number int64 + PosterID int64 `yaml:"poster_id"` + PosterName string `yaml:"poster_name"` + PosterEmail string `yaml:"poster_email"` + Title string + Content string + Ref string + Milestone string + State string // closed, open + IsLocked bool `yaml:"is_locked"` + Created time.Time + Updated time.Time + Closed *time.Time + Labels []*Label + Reactions []*Reaction + Assignees []string + Context IssueContext `yaml:"-"` +} diff --git a/modules/migration/label.go b/modules/migration/label.go new file mode 100644 index 0000000000..1a04a1dd3a --- /dev/null +++ b/modules/migration/label.go @@ -0,0 +1,13 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +// Label defines a standard label information +type Label struct { + Name string + Color string + Description string +} diff --git a/modules/migration/messenger.go b/modules/migration/messenger.go new file mode 100644 index 0000000000..fa8218cf93 --- /dev/null +++ b/modules/migration/messenger.go @@ -0,0 +1,11 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +// Messenger is a formatting function similar to i18n.Tr +type Messenger func(key string, args ...interface{}) + +// NilMessenger represents an empty formatting function +func NilMessenger(string, ...interface{}) {} diff --git a/modules/migration/milestone.go b/modules/migration/milestone.go new file mode 100644 index 0000000000..209aafe6a7 --- /dev/null +++ b/modules/migration/milestone.go @@ -0,0 +1,19 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import "time" + +// Milestone defines a standard milestone +type Milestone struct { + Title string + Description string + Deadline *time.Time + Created time.Time + Updated *time.Time + Closed *time.Time + State string // open, closed +} diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go new file mode 100644 index 0000000000..05daf72108 --- /dev/null +++ b/modules/migration/null_downloader.go @@ -0,0 +1,87 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import ( + "context" + "net/url" +) + +// NullDownloader implements a blank downloader +type NullDownloader struct { +} + +var ( + _ Downloader = &NullDownloader{} +) + +// SetContext set context +func (n NullDownloader) SetContext(_ context.Context) {} + +// GetRepoInfo returns a repository information +func (n NullDownloader) GetRepoInfo() (*Repository, error) { + return nil, &ErrNotSupported{Entity: "RepoInfo"} +} + +// GetTopics return repository topics +func (n NullDownloader) GetTopics() ([]string, error) { + return nil, &ErrNotSupported{Entity: "Topics"} +} + +// GetMilestones returns milestones +func (n NullDownloader) GetMilestones() ([]*Milestone, error) { + return nil, &ErrNotSupported{Entity: "Milestones"} +} + +// GetReleases returns releases +func (n NullDownloader) GetReleases() ([]*Release, error) { + return nil, &ErrNotSupported{Entity: "Releases"} +} + +// GetLabels returns labels +func (n NullDownloader) GetLabels() ([]*Label, error) { + return nil, &ErrNotSupported{Entity: "Labels"} +} + +// GetIssues returns issues according start and limit +func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { + return nil, false, &ErrNotSupported{Entity: "Issues"} +} + +// GetComments returns comments according the options +func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) { + return nil, false, &ErrNotSupported{Entity: "Comments"} +} + +// GetPullRequests returns pull requests according page and perPage +func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { + return nil, false, &ErrNotSupported{Entity: "PullRequests"} +} + +// GetReviews returns pull requests review +func (n NullDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) { + return nil, &ErrNotSupported{Entity: "Reviews"} +} + +// FormatCloneURL add authentification into remote URLs +func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) { + if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 { + u, err := url.Parse(remoteAddr) + if err != nil { + return "", err + } + u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword) + if len(opts.AuthToken) > 0 { + u.User = url.UserPassword("oauth2", opts.AuthToken) + } + return u.String(), nil + } + return remoteAddr, nil +} + +// SupportGetRepoComments return true if it supports get repo comments +func (n NullDownloader) SupportGetRepoComments() bool { + return false +} diff --git a/modules/migration/options.go b/modules/migration/options.go new file mode 100644 index 0000000000..1e92a1b0b3 --- /dev/null +++ b/modules/migration/options.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import "code.gitea.io/gitea/modules/structs" + +// MigrateOptions defines the way a repository gets migrated +// this is for internal usage by migrations module and func who interact with it +type MigrateOptions struct { + // required: true + CloneAddr string `json:"clone_addr" binding:"Required"` + CloneAddrEncrypted string `json:"clone_addr_encrypted,omitempty"` + AuthUsername string `json:"auth_username"` + AuthPassword string `json:"-"` + AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"` + AuthToken string `json:"-"` + AuthTokenEncrypted string `json:"auth_token_encrypted,omitempty"` + // required: true + UID int `json:"uid" binding:"Required"` + // required: true + RepoName string `json:"repo_name" binding:"Required"` + Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` + LFSEndpoint string `json:"lfs_endpoint"` + Private bool `json:"private"` + Description string `json:"description"` + OriginalURL string + GitServiceType structs.GitServiceType + Wiki bool + Issues bool + Milestones bool + Labels bool + Releases bool + Comments bool + PullRequests bool + ReleaseAssets bool + MigrateToRepoID int64 + MirrorInterval string `json:"mirror_interval"` +} diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go new file mode 100644 index 0000000000..9ca9a70b7d --- /dev/null +++ b/modules/migration/pullrequest.go @@ -0,0 +1,61 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import ( + "fmt" + "time" +) + +// PullRequest defines a standard pull request information +type PullRequest struct { + Number int64 + Title string + PosterName string `yaml:"poster_name"` + PosterID int64 `yaml:"poster_id"` + PosterEmail string `yaml:"poster_email"` + Content string + Milestone string + State string + Created time.Time + Updated time.Time + Closed *time.Time + Labels []*Label + PatchURL string `yaml:"patch_url"` + Merged bool + MergedTime *time.Time `yaml:"merged_time"` + MergeCommitSHA string `yaml:"merge_commit_sha"` + Head PullRequestBranch + Base PullRequestBranch + Assignees []string + IsLocked bool `yaml:"is_locked"` + Reactions []*Reaction + Context IssueContext `yaml:"-"` +} + +// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository +func (p *PullRequest) IsForkPullRequest() bool { + return p.Head.RepoPath() != p.Base.RepoPath() +} + +// GetGitRefName returns pull request relative path to head +func (p PullRequest) GetGitRefName() string { + return fmt.Sprintf("refs/pull/%d/head", p.Number) +} + +// PullRequestBranch represents a pull request branch +type PullRequestBranch struct { + CloneURL string `yaml:"clone_url"` + Ref string + SHA string + RepoName string `yaml:"repo_name"` + OwnerName string `yaml:"owner_name"` +} + +// RepoPath returns pull request repo path +func (p PullRequestBranch) RepoPath() string { + return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName) +} diff --git a/modules/migration/reaction.go b/modules/migration/reaction.go new file mode 100644 index 0000000000..004cff2f94 --- /dev/null +++ b/modules/migration/reaction.go @@ -0,0 +1,12 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +// Reaction represents a reaction to an issue/pr/comment. +type Reaction struct { + UserID int64 `yaml:"user_id"` + UserName string `yaml:"user_name"` + Content string +} diff --git a/modules/migration/release.go b/modules/migration/release.go new file mode 100644 index 0000000000..a83f5502cb --- /dev/null +++ b/modules/migration/release.go @@ -0,0 +1,40 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import ( + "io" + "time" +) + +// ReleaseAsset represents a release asset +type ReleaseAsset struct { + ID int64 + Name string + ContentType *string `yaml:"content_type"` + Size *int + DownloadCount *int `yaml:"download_count"` + Created time.Time + Updated time.Time + DownloadURL *string `yaml:"download_url"` + // if DownloadURL is nil, the function should be invoked + DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` +} + +// Release represents a release +type Release struct { + TagName string `yaml:"tag_name"` + TargetCommitish string `yaml:"target_commitish"` + Name string + Body string + Draft bool + Prerelease bool + PublisherID int64 `yaml:"publisher_id"` + PublisherName string `yaml:"publisher_name"` + PublisherEmail string `yaml:"publisher_email"` + Assets []*ReleaseAsset + Created time.Time + Published time.Time +} diff --git a/modules/migration/repo.go b/modules/migration/repo.go new file mode 100644 index 0000000000..d0d62de8da --- /dev/null +++ b/modules/migration/repo.go @@ -0,0 +1,18 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +// Repository defines a standard repository information +type Repository struct { + Name string + Owner string + IsPrivate bool `yaml:"is_private"` + IsMirror bool `yaml:"is_mirror"` + Description string + CloneURL string `yaml:"clone_url"` + OriginalURL string `yaml:"original_url"` + DefaultBranch string +} diff --git a/modules/migration/retry_downloader.go b/modules/migration/retry_downloader.go new file mode 100644 index 0000000000..1f034ab0c7 --- /dev/null +++ b/modules/migration/retry_downloader.go @@ -0,0 +1,197 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import ( + "context" + "time" +) + +var ( + _ Downloader = &RetryDownloader{} +) + +// RetryDownloader retry the downloads +type RetryDownloader struct { + Downloader + ctx context.Context + RetryTimes int // the total execute times + RetryDelay int // time to delay seconds +} + +// NewRetryDownloader creates a retry downloader +func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader { + return &RetryDownloader{ + Downloader: downloader, + ctx: ctx, + RetryTimes: retryTimes, + RetryDelay: retryDelay, + } +} + +func (d *RetryDownloader) retry(work func() error) error { + var ( + times = d.RetryTimes + err error + ) + for ; times > 0; times-- { + if err = work(); err == nil { + return nil + } + if IsErrNotSupported(err) { + return err + } + select { + case <-d.ctx.Done(): + return d.ctx.Err() + case <-time.After(time.Second * time.Duration(d.RetryDelay)): + } + } + return err +} + +// SetContext set context +func (d *RetryDownloader) SetContext(ctx context.Context) { + d.ctx = ctx + d.Downloader.SetContext(ctx) +} + +// GetRepoInfo returns a repository information with retry +func (d *RetryDownloader) GetRepoInfo() (*Repository, error) { + var ( + repo *Repository + err error + ) + + err = d.retry(func() error { + repo, err = d.Downloader.GetRepoInfo() + return err + }) + + return repo, err +} + +// GetTopics returns a repository's topics with retry +func (d *RetryDownloader) GetTopics() ([]string, error) { + var ( + topics []string + err error + ) + + err = d.retry(func() error { + topics, err = d.Downloader.GetTopics() + return err + }) + + return topics, err +} + +// GetMilestones returns a repository's milestones with retry +func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) { + var ( + milestones []*Milestone + err error + ) + + err = d.retry(func() error { + milestones, err = d.Downloader.GetMilestones() + return err + }) + + return milestones, err +} + +// GetReleases returns a repository's releases with retry +func (d *RetryDownloader) GetReleases() ([]*Release, error) { + var ( + releases []*Release + err error + ) + + err = d.retry(func() error { + releases, err = d.Downloader.GetReleases() + return err + }) + + return releases, err +} + +// GetLabels returns a repository's labels with retry +func (d *RetryDownloader) GetLabels() ([]*Label, error) { + var ( + labels []*Label + err error + ) + + err = d.retry(func() error { + labels, err = d.Downloader.GetLabels() + return err + }) + + return labels, err +} + +// GetIssues returns a repository's issues with retry +func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { + var ( + issues []*Issue + isEnd bool + err error + ) + + err = d.retry(func() error { + issues, isEnd, err = d.Downloader.GetIssues(page, perPage) + return err + }) + + return issues, isEnd, err +} + +// GetComments returns a repository's comments with retry +func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) { + var ( + comments []*Comment + isEnd bool + err error + ) + + err = d.retry(func() error { + comments, isEnd, err = d.Downloader.GetComments(opts) + return err + }) + + return comments, isEnd, err +} + +// GetPullRequests returns a repository's pull requests with retry +func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) { + var ( + prs []*PullRequest + err error + isEnd bool + ) + + err = d.retry(func() error { + prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage) + return err + }) + + return prs, isEnd, err +} + +// GetReviews returns pull requests reviews +func (d *RetryDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) { + var ( + reviews []*Review + err error + ) + + err = d.retry(func() error { + reviews, err = d.Downloader.GetReviews(pullRequestContext) + return err + }) + + return reviews, err +} diff --git a/modules/migration/review.go b/modules/migration/review.go new file mode 100644 index 0000000000..d6d15002af --- /dev/null +++ b/modules/migration/review.go @@ -0,0 +1,45 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +import "time" + +// enumerate all review states +const ( + ReviewStatePending = "PENDING" + ReviewStateApproved = "APPROVED" + ReviewStateChangesRequested = "CHANGES_REQUESTED" + ReviewStateCommented = "COMMENTED" +) + +// Review is a standard review information +type Review struct { + ID int64 + IssueIndex int64 `yaml:"issue_index"` + ReviewerID int64 `yaml:"reviewer_id"` + ReviewerName string `yaml:"reviewer_name"` + Official bool + CommitID string `yaml:"commit_id"` + Content string + CreatedAt time.Time `yaml:"created_at"` + State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT + Comments []*ReviewComment +} + +// ReviewComment represents a review comment +type ReviewComment struct { + ID int64 + InReplyTo int64 `yaml:"in_reply_to"` + Content string + TreePath string `yaml:"tree_path"` + DiffHunk string `yaml:"diff_hunk"` + Position int + Line int + CommitID string `yaml:"commit_id"` + PosterID int64 `yaml:"poster_id"` + Reactions []*Reaction + CreatedAt time.Time `yaml:"created_at"` + UpdatedAt time.Time `yaml:"updated_at"` +} diff --git a/modules/migration/uploader.go b/modules/migration/uploader.go new file mode 100644 index 0000000000..57571861aa --- /dev/null +++ b/modules/migration/uploader.go @@ -0,0 +1,24 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2018 Jonas Franz. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +// Uploader uploads all the information of one repository +type Uploader interface { + MaxBatchInsertSize(tp string) int + CreateRepo(repo *Repository, opts MigrateOptions) error + CreateTopics(topic ...string) error + CreateMilestones(milestones ...*Milestone) error + CreateReleases(releases ...*Release) error + SyncTags() error + CreateLabels(labels ...*Label) error + CreateIssues(issues ...*Issue) error + CreateComments(comments ...*Comment) error + CreatePullRequests(prs ...*PullRequest) error + CreateReviews(reviews ...*Review) error + Rollback() error + Finish() error + Close() +} |