123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454 |
- // Copyright 2019 The Gitea Authors. All rights reserved.
- // Copyright 2018 Jonas Franz. All rights reserved.
- // Use of this source code is governed by a MIT-style
- // license that can be found in the LICENSE file.
-
- package migrations
-
- import (
- "context"
- "fmt"
- "net"
- "net/url"
- "path/filepath"
- "strings"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/matchlist"
- "code.gitea.io/gitea/modules/migrations/base"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/util"
- )
-
- // MigrateOptions is equal to base.MigrateOptions
- type MigrateOptions = base.MigrateOptions
-
- var (
- factories []base.DownloaderFactory
-
- allowList *matchlist.Matchlist
- blockList *matchlist.Matchlist
- )
-
- // RegisterDownloaderFactory registers a downloader factory
- func RegisterDownloaderFactory(factory base.DownloaderFactory) {
- factories = append(factories, factory)
- }
-
- // IsMigrateURLAllowed checks if an URL is allowed to be migrated from
- func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
- // Remote address can be HTTP/HTTPS/Git URL or local path.
- u, err := url.Parse(strings.ToLower(remoteURL))
- if err != nil {
- return &models.ErrInvalidCloneAddr{IsURLError: true}
- }
-
- if u.Scheme == "file" || u.Scheme == "" {
- if !doer.CanImportLocal() {
- return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsPermissionDenied: true, LocalPath: true}
- }
- isAbs := filepath.IsAbs(u.Host + u.Path)
- if !isAbs {
- return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
- }
- isDir, err := util.IsDir(u.Host + u.Path)
- if err != nil {
- log.Error("Unable to check if %s is a directory: %v", u.Host+u.Path, err)
- return err
- }
- if !isDir {
- return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
- }
-
- return nil
- }
-
- if u.Scheme == "git" && u.Port() != "" && (strings.Contains(remoteURL, "%0d") || strings.Contains(remoteURL, "%0a")) {
- return &models.ErrInvalidCloneAddr{Host: u.Host, IsURLError: true}
- }
-
- if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" {
- return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true}
- }
-
- if len(setting.Migrations.AllowedDomains) > 0 {
- if !allowList.Match(u.Host) {
- return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
- }
- } else {
- if blockList.Match(u.Host) {
- return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
- }
- }
-
- if !setting.Migrations.AllowLocalNetworks {
- addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0])
- if err != nil {
- return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true}
- }
- for _, addr := range addrList {
- if isIPPrivate(addr) || !addr.IsGlobalUnicast() {
- return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true}
- }
- }
- }
-
- return nil
- }
-
- // MigrateRepository migrate repository according MigrateOptions
- func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
- err := IsMigrateURLAllowed(opts.CloneAddr, doer)
- if err != nil {
- return nil, err
- }
- downloader, err := newDownloader(ctx, ownerName, opts)
- if err != nil {
- return nil, err
- }
-
- var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
- uploader.gitServiceType = opts.GitServiceType
-
- if err := migrateRepository(downloader, uploader, opts); err != nil {
- if err1 := uploader.Rollback(); err1 != nil {
- log.Error("rollback failed: %v", err1)
- }
- if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
- log.Error("create respotiry notice failed: ", err2)
- }
- return nil, err
- }
- return uploader.repo, nil
- }
-
- func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) {
- var (
- downloader base.Downloader
- err error
- )
-
- for _, factory := range factories {
- if factory.GitServiceType() == opts.GitServiceType {
- downloader, err = factory.New(ctx, opts)
- if err != nil {
- return nil, err
- }
- break
- }
- }
-
- if downloader == nil {
- opts.Wiki = true
- opts.Milestones = false
- opts.Labels = false
- opts.Releases = false
- opts.Comments = false
- opts.Issues = false
- opts.PullRequests = false
- downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
- log.Trace("Will migrate from git: %s", opts.OriginalURL)
- }
-
- if setting.Migrations.MaxAttempts > 1 {
- downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
- }
- return downloader, nil
- }
-
- // migrateRepository will download information and then upload it to Uploader, this is a simple
- // process for small repository. For a big repository, save all the data to disk
- // before upload is better
- func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error {
- repo, err := downloader.GetRepoInfo()
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Info("migrating repo infos is not supported, ignored")
- }
- repo.IsPrivate = opts.Private
- repo.IsMirror = opts.Mirror
- if opts.Description != "" {
- repo.Description = opts.Description
- }
- if repo.CloneURL, err = downloader.FormatCloneURL(opts, repo.CloneURL); err != nil {
- return err
- }
-
- log.Trace("migrating git data")
- if err = uploader.CreateRepo(repo, opts); err != nil {
- return err
- }
- defer uploader.Close()
-
- log.Trace("migrating topics")
- topics, err := downloader.GetTopics()
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating topics is not supported, ignored")
- }
- if len(topics) != 0 {
- if err = uploader.CreateTopics(topics...); err != nil {
- return err
- }
- }
-
- if opts.Milestones {
- log.Trace("migrating milestones")
- milestones, err := downloader.GetMilestones()
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating milestones is not supported, ignored")
- }
-
- msBatchSize := uploader.MaxBatchInsertSize("milestone")
- for len(milestones) > 0 {
- if len(milestones) < msBatchSize {
- msBatchSize = len(milestones)
- }
-
- if err := uploader.CreateMilestones(milestones...); err != nil {
- return err
- }
- milestones = milestones[msBatchSize:]
- }
- }
-
- if opts.Labels {
- log.Trace("migrating labels")
- labels, err := downloader.GetLabels()
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating labels is not supported, ignored")
- }
-
- lbBatchSize := uploader.MaxBatchInsertSize("label")
- for len(labels) > 0 {
- if len(labels) < lbBatchSize {
- lbBatchSize = len(labels)
- }
-
- if err := uploader.CreateLabels(labels...); err != nil {
- return err
- }
- labels = labels[lbBatchSize:]
- }
- }
-
- if opts.Releases {
- log.Trace("migrating releases")
- releases, err := downloader.GetReleases()
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating releases is not supported, ignored")
- }
-
- relBatchSize := uploader.MaxBatchInsertSize("release")
- for len(releases) > 0 {
- if len(releases) < relBatchSize {
- relBatchSize = len(releases)
- }
-
- if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
- return err
- }
- releases = releases[relBatchSize:]
- }
-
- // Once all releases (if any) are inserted, sync any remaining non-release tags
- if err = uploader.SyncTags(); err != nil {
- return err
- }
- }
-
- var (
- commentBatchSize = uploader.MaxBatchInsertSize("comment")
- reviewBatchSize = uploader.MaxBatchInsertSize("review")
- )
-
- if opts.Issues {
- log.Trace("migrating issues and comments")
- var issueBatchSize = uploader.MaxBatchInsertSize("issue")
-
- for i := 1; ; i++ {
- issues, isEnd, err := downloader.GetIssues(i, issueBatchSize)
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating issues is not supported, ignored")
- break
- }
-
- if err := uploader.CreateIssues(issues...); err != nil {
- return err
- }
-
- if opts.Comments {
- var allComments = make([]*base.Comment, 0, commentBatchSize)
- for _, issue := range issues {
- log.Trace("migrating issue %d's comments", issue.Number)
- comments, err := downloader.GetComments(issue.Number)
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating comments is not supported, ignored")
- }
-
- allComments = append(allComments, comments...)
-
- if len(allComments) >= commentBatchSize {
- if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
- return err
- }
-
- allComments = allComments[commentBatchSize:]
- }
- }
-
- if len(allComments) > 0 {
- if err = uploader.CreateComments(allComments...); err != nil {
- return err
- }
- }
- }
-
- if isEnd {
- break
- }
- }
- }
-
- if opts.PullRequests {
- log.Trace("migrating pull requests and comments")
- var prBatchSize = uploader.MaxBatchInsertSize("pullrequest")
- for i := 1; ; i++ {
- prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating pull requests is not supported, ignored")
- break
- }
-
- if err := uploader.CreatePullRequests(prs...); err != nil {
- return err
- }
-
- if opts.Comments {
- // plain comments
- var allComments = make([]*base.Comment, 0, commentBatchSize)
- for _, pr := range prs {
- log.Trace("migrating pull request %d's comments", pr.Number)
- comments, err := downloader.GetComments(pr.Number)
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating comments is not supported, ignored")
- }
-
- allComments = append(allComments, comments...)
-
- if len(allComments) >= commentBatchSize {
- if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
- return err
- }
- allComments = allComments[commentBatchSize:]
- }
- }
- if len(allComments) > 0 {
- if err = uploader.CreateComments(allComments...); err != nil {
- return err
- }
- }
-
- // migrate reviews
- var allReviews = make([]*base.Review, 0, reviewBatchSize)
- for _, pr := range prs {
- number := pr.Number
-
- // on gitlab migrations pull number change
- if pr.OriginalNumber > 0 {
- number = pr.OriginalNumber
- }
-
- reviews, err := downloader.GetReviews(number)
- if err != nil {
- if !base.IsErrNotSupported(err) {
- return err
- }
- log.Warn("migrating reviews is not supported, ignored")
- break
- }
- if pr.OriginalNumber > 0 {
- for i := range reviews {
- reviews[i].IssueIndex = pr.Number
- }
- }
-
- allReviews = append(allReviews, reviews...)
-
- if len(allReviews) >= reviewBatchSize {
- if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
- return err
- }
- allReviews = allReviews[reviewBatchSize:]
- }
- }
- if len(allReviews) > 0 {
- if err = uploader.CreateReviews(allReviews...); err != nil {
- return err
- }
- }
- }
-
- if isEnd {
- break
- }
- }
- }
-
- return uploader.Finish()
- }
-
- // Init migrations service
- func Init() error {
- var err error
- allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...)
- if err != nil {
- return fmt.Errorf("init migration allowList domains failed: %v", err)
- }
-
- blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...)
- if err != nil {
- return fmt.Errorf("init migration blockList domains failed: %v", err)
- }
-
- return nil
- }
-
- // isIPPrivate reports whether ip is a private address, according to
- // RFC 1918 (IPv4 addresses) and RFC 4193 (IPv6 addresses).
- // from https://github.com/golang/go/pull/42793
- // TODO remove if https://github.com/golang/go/issues/29146 got resolved
- func isIPPrivate(ip net.IP) bool {
- if ip4 := ip.To4(); ip4 != nil {
- return ip4[0] == 10 ||
- (ip4[0] == 172 && ip4[1]&0xf0 == 16) ||
- (ip4[0] == 192 && ip4[1] == 168)
- }
- return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc
- }
|