Move migrations into services and base into modules/migration (#17663)

* Move migrtions into services and base into modules/migration

* Fix imports

* Fix lint

1 files changed, 476 insertions, 0 deletions

diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go
new file mode 100644
index 0000000000..21be2d3b0d
--- /dev/null
+++ b/services/migrations/migrate.go
@@ -0,0 +1,476 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/url"
+	"path/filepath"
+	"strings"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/matchlist"
+	base "code.gitea.io/gitea/modules/migration"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+)
+
+// MigrateOptions is equal to base.MigrateOptions
+type MigrateOptions = base.MigrateOptions
+
+var (
+	factories []base.DownloaderFactory
+
+	allowList *matchlist.Matchlist
+	blockList *matchlist.Matchlist
+)
+
+// RegisterDownloaderFactory registers a downloader factory
+func RegisterDownloaderFactory(factory base.DownloaderFactory) {
+	factories = append(factories, factory)
+}
+
+// IsMigrateURLAllowed checks if an URL is allowed to be migrated from
+func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
+	// Remote address can be HTTP/HTTPS/Git URL or local path.
+	u, err := url.Parse(remoteURL)
+	if err != nil {
+		return &models.ErrInvalidCloneAddr{IsURLError: true}
+	}
+
+	if u.Scheme == "file" || u.Scheme == "" {
+		if !doer.CanImportLocal() {
+			return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsPermissionDenied: true, LocalPath: true}
+		}
+		isAbs := filepath.IsAbs(u.Host + u.Path)
+		if !isAbs {
+			return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
+		}
+		isDir, err := util.IsDir(u.Host + u.Path)
+		if err != nil {
+			log.Error("Unable to check if %s is a directory: %v", u.Host+u.Path, err)
+			return err
+		}
+		if !isDir {
+			return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
+		}
+
+		return nil
+	}
+
+	if u.Scheme == "git" && u.Port() != "" && (strings.Contains(remoteURL, "%0d") || strings.Contains(remoteURL, "%0a")) {
+		return &models.ErrInvalidCloneAddr{Host: u.Host, IsURLError: true}
+	}
+
+	if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" {
+		return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true}
+	}
+
+	host := strings.ToLower(u.Host)
+	if len(setting.Migrations.AllowedDomains) > 0 {
+		if !allowList.Match(host) {
+			return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
+		}
+	} else {
+		if blockList.Match(host) {
+			return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
+		}
+	}
+
+	if !setting.Migrations.AllowLocalNetworks {
+		addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0])
+		if err != nil {
+			return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true}
+		}
+		for _, addr := range addrList {
+			if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() {
+				return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true}
+			}
+		}
+	}
+
+	return nil
+}
+
+// MigrateRepository migrate repository according MigrateOptions
+func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*models.Repository, error) {
+	err := IsMigrateURLAllowed(opts.CloneAddr, doer)
+	if err != nil {
+		return nil, err
+	}
+	if opts.LFS && len(opts.LFSEndpoint) > 0 {
+		err := IsMigrateURLAllowed(opts.LFSEndpoint, doer)
+		if err != nil {
+			return nil, err
+		}
+	}
+	downloader, err := newDownloader(ctx, ownerName, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
+	uploader.gitServiceType = opts.GitServiceType
+
+	if err := migrateRepository(downloader, uploader, opts, messenger); err != nil {
+		if err1 := uploader.Rollback(); err1 != nil {
+			log.Error("rollback failed: %v", err1)
+		}
+		if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
+			log.Error("create respotiry notice failed: ", err2)
+		}
+		return nil, err
+	}
+	return uploader.repo, nil
+}
+
+func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) {
+	var (
+		downloader base.Downloader
+		err        error
+	)
+
+	for _, factory := range factories {
+		if factory.GitServiceType() == opts.GitServiceType {
+			downloader, err = factory.New(ctx, opts)
+			if err != nil {
+				return nil, err
+			}
+			break
+		}
+	}
+
+	if downloader == nil {
+		opts.Wiki = true
+		opts.Milestones = false
+		opts.Labels = false
+		opts.Releases = false
+		opts.Comments = false
+		opts.Issues = false
+		opts.PullRequests = false
+		downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
+		log.Trace("Will migrate from git: %s", opts.OriginalURL)
+	}
+
+	if setting.Migrations.MaxAttempts > 1 {
+		downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
+	}
+	return downloader, nil
+}
+
+// migrateRepository will download information and then upload it to Uploader, this is a simple
+// process for small repository. For a big repository, save all the data to disk
+// before upload is better
+func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
+	if messenger == nil {
+		messenger = base.NilMessenger
+	}
+
+	repo, err := downloader.GetRepoInfo()
+	if err != nil {
+		if !base.IsErrNotSupported(err) {
+			return err
+		}
+		log.Info("migrating repo infos is not supported, ignored")
+	}
+	repo.IsPrivate = opts.Private
+	repo.IsMirror = opts.Mirror
+	if opts.Description != "" {
+		repo.Description = opts.Description
+	}
+	if repo.CloneURL, err = downloader.FormatCloneURL(opts, repo.CloneURL); err != nil {
+		return err
+	}
+
+	log.Trace("migrating git data from %s", repo.CloneURL)
+	messenger("repo.migrate.migrating_git")
+	if err = uploader.CreateRepo(repo, opts); err != nil {
+		return err
+	}
+	defer uploader.Close()
+
+	log.Trace("migrating topics")
+	messenger("repo.migrate.migrating_topics")
+	topics, err := downloader.GetTopics()
+	if err != nil {
+		if !base.IsErrNotSupported(err) {
+			return err
+		}
+		log.Warn("migrating topics is not supported, ignored")
+	}
+	if len(topics) != 0 {
+		if err = uploader.CreateTopics(topics...); err != nil {
+			return err
+		}
+	}
+
+	if opts.Milestones {
+		log.Trace("migrating milestones")
+		messenger("repo.migrate.migrating_milestones")
+		milestones, err := downloader.GetMilestones()
+		if err != nil {
+			if !base.IsErrNotSupported(err) {
+				return err
+			}
+			log.Warn("migrating milestones is not supported, ignored")
+		}
+
+		msBatchSize := uploader.MaxBatchInsertSize("milestone")
+		for len(milestones) > 0 {
+			if len(milestones) < msBatchSize {
+				msBatchSize = len(milestones)
+			}
+
+			if err := uploader.CreateMilestones(milestones...); err != nil {
+				return err
+			}
+			milestones = milestones[msBatchSize:]
+		}
+	}
+
+	if opts.Labels {
+		log.Trace("migrating labels")
+		messenger("repo.migrate.migrating_labels")
+		labels, err := downloader.GetLabels()
+		if err != nil {
+			if !base.IsErrNotSupported(err) {
+				return err
+			}
+			log.Warn("migrating labels is not supported, ignored")
+		}
+
+		lbBatchSize := uploader.MaxBatchInsertSize("label")
+		for len(labels) > 0 {
+			if len(labels) < lbBatchSize {
+				lbBatchSize = len(labels)
+			}
+
+			if err := uploader.CreateLabels(labels...); err != nil {
+				return err
+			}
+			labels = labels[lbBatchSize:]
+		}
+	}
+
+	if opts.Releases {
+		log.Trace("migrating releases")
+		messenger("repo.migrate.migrating_releases")
+		releases, err := downloader.GetReleases()
+		if err != nil {
+			if !base.IsErrNotSupported(err) {
+				return err
+			}
+			log.Warn("migrating releases is not supported, ignored")
+		}
+
+		relBatchSize := uploader.MaxBatchInsertSize("release")
+		for len(releases) > 0 {
+			if len(releases) < relBatchSize {
+				relBatchSize = len(releases)
+			}
+
+			if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
+				return err
+			}
+			releases = releases[relBatchSize:]
+		}
+
+		// Once all releases (if any) are inserted, sync any remaining non-release tags
+		if err = uploader.SyncTags(); err != nil {
+			return err
+		}
+	}
+
+	var (
+		commentBatchSize = uploader.MaxBatchInsertSize("comment")
+		reviewBatchSize  = uploader.MaxBatchInsertSize("review")
+	)
+
+	supportAllComments := downloader.SupportGetRepoComments()
+
+	if opts.Issues {
+		log.Trace("migrating issues and comments")
+		messenger("repo.migrate.migrating_issues")
+		var issueBatchSize = uploader.MaxBatchInsertSize("issue")
+
+		for i := 1; ; i++ {
+			issues, isEnd, err := downloader.GetIssues(i, issueBatchSize)
+			if err != nil {
+				if !base.IsErrNotSupported(err) {
+					return err
+				}
+				log.Warn("migrating issues is not supported, ignored")
+				break
+			}
+
+			if err := uploader.CreateIssues(issues...); err != nil {
+				return err
+			}
+
+			if opts.Comments && !supportAllComments {
+				var allComments = make([]*base.Comment, 0, commentBatchSize)
+				for _, issue := range issues {
+					log.Trace("migrating issue %d's comments", issue.Number)
+					comments, _, err := downloader.GetComments(base.GetCommentOptions{
+						Context: issue.Context,
+					})
+					if err != nil {
+						if !base.IsErrNotSupported(err) {
+							return err
+						}
+						log.Warn("migrating comments is not supported, ignored")
+					}
+
+					allComments = append(allComments, comments...)
+
+					if len(allComments) >= commentBatchSize {
+						if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
+							return err
+						}
+
+						allComments = allComments[commentBatchSize:]
+					}
+				}
+
+				if len(allComments) > 0 {
+					if err = uploader.CreateComments(allComments...); err != nil {
+						return err
+					}
+				}
+			}
+
+			if isEnd {
+				break
+			}
+		}
+	}
+
+	if opts.PullRequests {
+		log.Trace("migrating pull requests and comments")
+		messenger("repo.migrate.migrating_pulls")
+		var prBatchSize = uploader.MaxBatchInsertSize("pullrequest")
+		for i := 1; ; i++ {
+			prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)
+			if err != nil {
+				if !base.IsErrNotSupported(err) {
+					return err
+				}
+				log.Warn("migrating pull requests is not supported, ignored")
+				break
+			}
+
+			if err := uploader.CreatePullRequests(prs...); err != nil {
+				return err
+			}
+
+			if opts.Comments {
+				if !supportAllComments {
+					// plain comments
+					var allComments = make([]*base.Comment, 0, commentBatchSize)
+					for _, pr := range prs {
+						log.Trace("migrating pull request %d's comments", pr.Number)
+						comments, _, err := downloader.GetComments(base.GetCommentOptions{
+							Context: pr.Context,
+						})
+						if err != nil {
+							if !base.IsErrNotSupported(err) {
+								return err
+							}
+							log.Warn("migrating comments is not supported, ignored")
+						}
+
+						allComments = append(allComments, comments...)
+
+						if len(allComments) >= commentBatchSize {
+							if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
+								return err
+							}
+							allComments = allComments[commentBatchSize:]
+						}
+					}
+					if len(allComments) > 0 {
+						if err = uploader.CreateComments(allComments...); err != nil {
+							return err
+						}
+					}
+				}
+
+				// migrate reviews
+				var allReviews = make([]*base.Review, 0, reviewBatchSize)
+				for _, pr := range prs {
+					reviews, err := downloader.GetReviews(pr.Context)
+					if err != nil {
+						if !base.IsErrNotSupported(err) {
+							return err
+						}
+						log.Warn("migrating reviews is not supported, ignored")
+						break
+					}
+
+					allReviews = append(allReviews, reviews...)
+
+					if len(allReviews) >= reviewBatchSize {
+						if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
+							return err
+						}
+						allReviews = allReviews[reviewBatchSize:]
+					}
+				}
+				if len(allReviews) > 0 {
+					if err = uploader.CreateReviews(allReviews...); err != nil {
+						return err
+					}
+				}
+			}
+
+			if isEnd {
+				break
+			}
+		}
+	}
+
+	if opts.Comments && supportAllComments {
+		log.Trace("migrating comments")
+		for i := 1; ; i++ {
+			comments, isEnd, err := downloader.GetComments(base.GetCommentOptions{
+				Page:     i,
+				PageSize: commentBatchSize,
+			})
+			if err != nil {
+				return err
+			}
+
+			if err := uploader.CreateComments(comments...); err != nil {
+				return err
+			}
+
+			if isEnd {
+				break
+			}
+		}
+	}
+
+	return uploader.Finish()
+}
+
+// Init migrations service
+func Init() error {
+	var err error
+	allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...)
+	if err != nil {
+		return fmt.Errorf("init migration allowList domains failed: %v", err)
+	}
+
+	blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...)
+	if err != nil {
+		return fmt.Errorf("init migration blockList domains failed: %v", err)
+	}
+
+	return nil
+}