diff options
Diffstat (limited to 'modules/migrations/migrate.go')
| -rw-r--r-- | modules/migrations/migrate.go | 476 |
1 files changed, 0 insertions, 476 deletions
diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go deleted file mode 100644 index dbe69259f4..0000000000 --- a/modules/migrations/migrate.go +++ /dev/null @@ -1,476 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// Copyright 2018 Jonas Franz. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package migrations - -import ( - "context" - "fmt" - "net" - "net/url" - "path/filepath" - "strings" - - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/matchlist" - "code.gitea.io/gitea/modules/migrations/base" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" -) - -// MigrateOptions is equal to base.MigrateOptions -type MigrateOptions = base.MigrateOptions - -var ( - factories []base.DownloaderFactory - - allowList *matchlist.Matchlist - blockList *matchlist.Matchlist -) - -// RegisterDownloaderFactory registers a downloader factory -func RegisterDownloaderFactory(factory base.DownloaderFactory) { - factories = append(factories, factory) -} - -// IsMigrateURLAllowed checks if an URL is allowed to be migrated from -func IsMigrateURLAllowed(remoteURL string, doer *models.User) error { - // Remote address can be HTTP/HTTPS/Git URL or local path. - u, err := url.Parse(remoteURL) - if err != nil { - return &models.ErrInvalidCloneAddr{IsURLError: true} - } - - if u.Scheme == "file" || u.Scheme == "" { - if !doer.CanImportLocal() { - return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsPermissionDenied: true, LocalPath: true} - } - isAbs := filepath.IsAbs(u.Host + u.Path) - if !isAbs { - return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true} - } - isDir, err := util.IsDir(u.Host + u.Path) - if err != nil { - log.Error("Unable to check if %s is a directory: %v", u.Host+u.Path, err) - return err - } - if !isDir { - return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true} - } - - return nil - } - - if u.Scheme == "git" && u.Port() != "" && (strings.Contains(remoteURL, "%0d") || strings.Contains(remoteURL, "%0a")) { - return &models.ErrInvalidCloneAddr{Host: u.Host, IsURLError: true} - } - - if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" { - return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true} - } - - host := strings.ToLower(u.Host) - if len(setting.Migrations.AllowedDomains) > 0 { - if !allowList.Match(host) { - return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true} - } - } else { - if blockList.Match(host) { - return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true} - } - } - - if !setting.Migrations.AllowLocalNetworks { - addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0]) - if err != nil { - return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true} - } - for _, addr := range addrList { - if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() { - return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true} - } - } - } - - return nil -} - -// MigrateRepository migrate repository according MigrateOptions -func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*models.Repository, error) { - err := IsMigrateURLAllowed(opts.CloneAddr, doer) - if err != nil { - return nil, err - } - if opts.LFS && len(opts.LFSEndpoint) > 0 { - err := IsMigrateURLAllowed(opts.LFSEndpoint, doer) - if err != nil { - return nil, err - } - } - downloader, err := newDownloader(ctx, ownerName, opts) - if err != nil { - return nil, err - } - - var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) - uploader.gitServiceType = opts.GitServiceType - - if err := migrateRepository(downloader, uploader, opts, messenger); err != nil { - if err1 := uploader.Rollback(); err1 != nil { - log.Error("rollback failed: %v", err1) - } - if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { - log.Error("create respotiry notice failed: ", err2) - } - return nil, err - } - return uploader.repo, nil -} - -func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) { - var ( - downloader base.Downloader - err error - ) - - for _, factory := range factories { - if factory.GitServiceType() == opts.GitServiceType { - downloader, err = factory.New(ctx, opts) - if err != nil { - return nil, err - } - break - } - } - - if downloader == nil { - opts.Wiki = true - opts.Milestones = false - opts.Labels = false - opts.Releases = false - opts.Comments = false - opts.Issues = false - opts.PullRequests = false - downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr) - log.Trace("Will migrate from git: %s", opts.OriginalURL) - } - - if setting.Migrations.MaxAttempts > 1 { - downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff) - } - return downloader, nil -} - -// migrateRepository will download information and then upload it to Uploader, this is a simple -// process for small repository. For a big repository, save all the data to disk -// before upload is better -func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error { - if messenger == nil { - messenger = base.NilMessenger - } - - repo, err := downloader.GetRepoInfo() - if err != nil { - if !base.IsErrNotSupported(err) { - return err - } - log.Info("migrating repo infos is not supported, ignored") - } - repo.IsPrivate = opts.Private - repo.IsMirror = opts.Mirror - if opts.Description != "" { - repo.Description = opts.Description - } - if repo.CloneURL, err = downloader.FormatCloneURL(opts, repo.CloneURL); err != nil { - return err - } - - log.Trace("migrating git data from %s", repo.CloneURL) - messenger("repo.migrate.migrating_git") - if err = uploader.CreateRepo(repo, opts); err != nil { - return err - } - defer uploader.Close() - - log.Trace("migrating topics") - messenger("repo.migrate.migrating_topics") - topics, err := downloader.GetTopics() - if err != nil { - if !base.IsErrNotSupported(err) { - return err - } - log.Warn("migrating topics is not supported, ignored") - } - if len(topics) != 0 { - if err = uploader.CreateTopics(topics...); err != nil { - return err - } - } - - if opts.Milestones { - log.Trace("migrating milestones") - messenger("repo.migrate.migrating_milestones") - milestones, err := downloader.GetMilestones() - if err != nil { - if !base.IsErrNotSupported(err) { - return err - } - log.Warn("migrating milestones is not supported, ignored") - } - - msBatchSize := uploader.MaxBatchInsertSize("milestone") - for len(milestones) > 0 { - if len(milestones) < msBatchSize { - msBatchSize = len(milestones) - } - - if err := uploader.CreateMilestones(milestones...); err != nil { - return err - } - milestones = milestones[msBatchSize:] - } - } - - if opts.Labels { - log.Trace("migrating labels") - messenger("repo.migrate.migrating_labels") - labels, err := downloader.GetLabels() - if err != nil { - if !base.IsErrNotSupported(err) { - return err - } - log.Warn("migrating labels is not supported, ignored") - } - - lbBatchSize := uploader.MaxBatchInsertSize("label") - for len(labels) > 0 { - if len(labels) < lbBatchSize { - lbBatchSize = len(labels) - } - - if err := uploader.CreateLabels(labels...); err != nil { - return err - } - labels = labels[lbBatchSize:] - } - } - - if opts.Releases { - log.Trace("migrating releases") - messenger("repo.migrate.migrating_releases") - releases, err := downloader.GetReleases() - if err != nil { - if !base.IsErrNotSupported(err) { - return err - } - log.Warn("migrating releases is not supported, ignored") - } - - relBatchSize := uploader.MaxBatchInsertSize("release") - for len(releases) > 0 { - if len(releases) < relBatchSize { - relBatchSize = len(releases) - } - - if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil { - return err - } - releases = releases[relBatchSize:] - } - - // Once all releases (if any) are inserted, sync any remaining non-release tags - if err = uploader.SyncTags(); err != nil { - return err - } - } - - var ( - commentBatchSize = uploader.MaxBatchInsertSize("comment") - reviewBatchSize = uploader.MaxBatchInsertSize("review") - ) - - supportAllComments := downloader.SupportGetRepoComments() - - if opts.Issues { - log.Trace("migrating issues and comments") - messenger("repo.migrate.migrating_issues") - var issueBatchSize = uploader.MaxBatchInsertSize("issue") - - for i := 1; ; i++ { - issues, isEnd, err := downloader.GetIssues(i, issueBatchSize) - if err != nil { - if !base.IsErrNotSupported(err) { - return err - } - log.Warn("migrating issues is not supported, ignored") - break - } - - if err := uploader.CreateIssues(issues...); err != nil { - return err - } - - if opts.Comments && !supportAllComments { - var allComments = make([]*base.Comment, 0, commentBatchSize) - for _, issue := range issues { - log.Trace("migrating issue %d's comments", issue.Number) - comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: issue.Context, - }) - if err != nil { - if !base.IsErrNotSupported(err) { - return err - } - log.Warn("migrating comments is not supported, ignored") - } - - allComments = append(allComments, comments...) - - if len(allComments) >= commentBatchSize { - if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { - return err - } - - allComments = allComments[commentBatchSize:] - } - } - - if len(allComments) > 0 { - if err = uploader.CreateComments(allComments...); err != nil { - return err - } - } - } - - if isEnd { - break - } - } - } - - if opts.PullRequests { - log.Trace("migrating pull requests and comments") - messenger("repo.migrate.migrating_pulls") - var prBatchSize = uploader.MaxBatchInsertSize("pullrequest") - for i := 1; ; i++ { - prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize) - if err != nil { - if !base.IsErrNotSupported(err) { - return err - } - log.Warn("migrating pull requests is not supported, ignored") - break - } - - if err := uploader.CreatePullRequests(prs...); err != nil { - return err - } - - if opts.Comments { - if !supportAllComments { - // plain comments - var allComments = make([]*base.Comment, 0, commentBatchSize) - for _, pr := range prs { - log.Trace("migrating pull request %d's comments", pr.Number) - comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: pr.Context, - }) - if err != nil { - if !base.IsErrNotSupported(err) { - return err - } - log.Warn("migrating comments is not supported, ignored") - } - - allComments = append(allComments, comments...) - - if len(allComments) >= commentBatchSize { - if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { - return err - } - allComments = allComments[commentBatchSize:] - } - } - if len(allComments) > 0 { - if err = uploader.CreateComments(allComments...); err != nil { - return err - } - } - } - - // migrate reviews - var allReviews = make([]*base.Review, 0, reviewBatchSize) - for _, pr := range prs { - reviews, err := downloader.GetReviews(pr.Context) - if err != nil { - if !base.IsErrNotSupported(err) { - return err - } - log.Warn("migrating reviews is not supported, ignored") - break - } - - allReviews = append(allReviews, reviews...) - - if len(allReviews) >= reviewBatchSize { - if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil { - return err - } - allReviews = allReviews[reviewBatchSize:] - } - } - if len(allReviews) > 0 { - if err = uploader.CreateReviews(allReviews...); err != nil { - return err - } - } - } - - if isEnd { - break - } - } - } - - if opts.Comments && supportAllComments { - log.Trace("migrating comments") - for i := 1; ; i++ { - comments, isEnd, err := downloader.GetComments(base.GetCommentOptions{ - Page: i, - PageSize: commentBatchSize, - }) - if err != nil { - return err - } - - if err := uploader.CreateComments(comments...); err != nil { - return err - } - - if isEnd { - break - } - } - } - - return uploader.Finish() -} - -// Init migrations service -func Init() error { - var err error - allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...) - if err != nil { - return fmt.Errorf("init migration allowList domains failed: %v", err) - } - - blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...) - if err != nil { - return fmt.Errorf("init migration blockList domains failed: %v", err) - } - - return nil -} |