Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)

* Dump github/gitlab repository data to a local directory

* Fix lint

* Adjust directory structure

* Allow migration special units

* Allow migration ignore release assets

* Fix lint

* Add restore repository

* stage the changes

* Merge

* Fix lint

* Update the interface

* Add some restore methods

* Finish restore

* Add comments

* Fix restore

* Add a token flag

* Fix bug

* Fix test

* Fix test

* Fix bug

* Fix bug

* Fix lint

* Fix restore

* refactor downloader

* fmt

* Fix bug isEnd detection on getIssues

* Refactor maxPerPage

* Remove unused codes

* Remove unused codes

* Fix bug

* Fix restore

* Fix dump

* Uploader should not depend downloader

* use release attachment name but not id

* Fix restore bug

* Fix lint

* Fix restore bug

* Add a method of DownloadFunc for base.Release to make uploader not depend on downloader

* fix Release yml marshal

* Fix trace information

* Fix bug when dump & restore

* Save relative path on yml file

* Fix bug

* Use relative path

* Update docs

* Use git service string but not int

* Recognize clone addr to service type

1 files changed, 87 insertions, 83 deletions

diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go
index b3ecb8114a..4c15626e57 100644
--- a/modules/migrations/migrate.go
+++ b/modules/migrations/migrate.go
@@ -73,10 +73,30 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
 	if err != nil {
 		return nil, err
 	}
+	downloader, err := newDownloader(ctx, ownerName, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
+	uploader.gitServiceType = opts.GitServiceType
+
+	if err := migrateRepository(downloader, uploader, opts); err != nil {
+		if err1 := uploader.Rollback(); err1 != nil {
+			log.Error("rollback failed: %v", err1)
+		}
+		if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
+			log.Error("create respotiry notice failed: ", err2)
+		}
+		return nil, err
+	}
+	return uploader.repo, nil
+}
 
+func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) {
 	var (
 		downloader base.Downloader
-		uploader   = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
+		err        error
 	)
 
 	for _, factory := range factories {
@@ -101,24 +121,10 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
 		log.Trace("Will migrate from git: %s", opts.OriginalURL)
 	}
 
-	uploader.gitServiceType = opts.GitServiceType
-
 	if setting.Migrations.MaxAttempts > 1 {
 		downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
 	}
-
-	if err := migrateRepository(downloader, uploader, opts); err != nil {
-		if err1 := uploader.Rollback(); err1 != nil {
-			log.Error("rollback failed: %v", err1)
-		}
-
-		if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
-			log.Error("create repository notice failed: ", err2)
-		}
-		return nil, err
-	}
-
-	return uploader.repo, nil
+	return downloader, nil
 }
 
 // migrateRepository will download information and then upload it to Uploader, this is a simple
@@ -204,7 +210,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				relBatchSize = len(releases)
 			}
 
-			if err := uploader.CreateReleases(downloader, releases[:relBatchSize]...); err != nil {
+			if err := uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
 				return err
 			}
 			releases = releases[relBatchSize:]
@@ -235,31 +241,30 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				return err
 			}
 
-			if !opts.Comments {
-				continue
-			}
+			if opts.Comments {
+				var allComments = make([]*base.Comment, 0, commentBatchSize)
+				for _, issue := range issues {
+					log.Trace("migrating issue %d's comments", issue.Number)
+					comments, err := downloader.GetComments(issue.Number)
+					if err != nil {
+						return err
+					}
 
-			var allComments = make([]*base.Comment, 0, commentBatchSize)
-			for _, issue := range issues {
-				comments, err := downloader.GetComments(issue.Number)
-				if err != nil {
-					return err
-				}
+					allComments = append(allComments, comments...)
 
-				allComments = append(allComments, comments...)
+					if len(allComments) >= commentBatchSize {
+						if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
+							return err
+						}
 
-				if len(allComments) >= commentBatchSize {
-					if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
-						return err
+						allComments = allComments[commentBatchSize:]
 					}
-
-					allComments = allComments[commentBatchSize:]
 				}
-			}
 
-			if len(allComments) > 0 {
-				if err := uploader.CreateComments(allComments...); err != nil {
-					return err
+				if len(allComments) > 0 {
+					if err := uploader.CreateComments(allComments...); err != nil {
+						return err
+					}
 				}
 			}
 
@@ -282,65 +287,64 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				return err
 			}
 
-			if !opts.Comments {
-				continue
-			}
-
-			// plain comments
-			var allComments = make([]*base.Comment, 0, commentBatchSize)
-			for _, pr := range prs {
-				comments, err := downloader.GetComments(pr.Number)
-				if err != nil {
-					return err
-				}
+			if opts.Comments {
+				// plain comments
+				var allComments = make([]*base.Comment, 0, commentBatchSize)
+				for _, pr := range prs {
+					log.Trace("migrating pull request %d's comments", pr.Number)
+					comments, err := downloader.GetComments(pr.Number)
+					if err != nil {
+						return err
+					}
 
-				allComments = append(allComments, comments...)
+					allComments = append(allComments, comments...)
 
-				if len(allComments) >= commentBatchSize {
-					if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
-						return err
+					if len(allComments) >= commentBatchSize {
+						if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
+							return err
+						}
+						allComments = allComments[commentBatchSize:]
 					}
-					allComments = allComments[commentBatchSize:]
 				}
-			}
-			if len(allComments) > 0 {
-				if err := uploader.CreateComments(allComments...); err != nil {
-					return err
+				if len(allComments) > 0 {
+					if err := uploader.CreateComments(allComments...); err != nil {
+						return err
+					}
 				}
-			}
 
-			// migrate reviews
-			var allReviews = make([]*base.Review, 0, reviewBatchSize)
-			for _, pr := range prs {
-				number := pr.Number
+				// migrate reviews
+				var allReviews = make([]*base.Review, 0, reviewBatchSize)
+				for _, pr := range prs {
+					number := pr.Number
 
-				// on gitlab migrations pull number change
-				if pr.OriginalNumber > 0 {
-					number = pr.OriginalNumber
-				}
+					// on gitlab migrations pull number change
+					if pr.OriginalNumber > 0 {
+						number = pr.OriginalNumber
+					}
 
-				reviews, err := downloader.GetReviews(number)
-				if pr.OriginalNumber > 0 {
-					for i := range reviews {
-						reviews[i].IssueIndex = pr.Number
+					reviews, err := downloader.GetReviews(number)
+					if pr.OriginalNumber > 0 {
+						for i := range reviews {
+							reviews[i].IssueIndex = pr.Number
+						}
+					}
+					if err != nil {
+						return err
 					}
-				}
-				if err != nil {
-					return err
-				}
 
-				allReviews = append(allReviews, reviews...)
+					allReviews = append(allReviews, reviews...)
 
-				if len(allReviews) >= reviewBatchSize {
-					if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
-						return err
+					if len(allReviews) >= reviewBatchSize {
+						if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
+							return err
+						}
+						allReviews = allReviews[reviewBatchSize:]
 					}
-					allReviews = allReviews[reviewBatchSize:]
 				}
-			}
-			if len(allReviews) > 0 {
-				if err := uploader.CreateReviews(allReviews...); err != nil {
-					return err
+				if len(allReviews) > 0 {
+					if err := uploader.CreateReviews(allReviews...); err != nil {
+						return err
+					}
 				}
 			}
 
@@ -350,7 +354,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 		}
 	}
 
-	return nil
+	return uploader.Finish()
 }
 
 // Init migrations service