Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)

* Dump github/gitlab repository data to a local directory

* Fix lint

* Adjust directory structure

* Allow migration special units

* Allow migration ignore release assets

* Fix lint

* Add restore repository

* stage the changes

* Merge

* Fix lint

* Update the interface

* Add some restore methods

* Finish restore

* Add comments

* Fix restore

* Add a token flag

* Fix bug

* Fix test

* Fix test

* Fix bug

* Fix bug

* Fix lint

* Fix restore

* refactor downloader

* fmt

* Fix bug isEnd detection on getIssues

* Refactor maxPerPage

* Remove unused codes

* Remove unused codes

* Fix bug

* Fix restore

* Fix dump

* Uploader should not depend downloader

* use release attachment name but not id

* Fix restore bug

* Fix lint

* Fix restore bug

* Add a method of DownloadFunc for base.Release to make uploader not depend on downloader

* fix Release yml marshal

* Fix trace information

* Fix bug when dump & restore

* Save relative path on yml file

* Fix bug

* Use relative path

* Update docs

* Use git service string but not int

* Recognize clone addr to service type

1 files changed, 39 insertions, 19 deletions

diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go
index 91ddda9c39..6118b3b5c1 100644
--- a/modules/migrations/gitea_uploader.go
+++ b/modules/migrations/gitea_uploader.go
@@ -10,7 +10,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -28,6 +27,7 @@ import (
 	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/uri"
 	"code.gitea.io/gitea/services/pull"
 
 	gouuid "github.com/google/uuid"
@@ -86,26 +86,33 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
 	return 10
 }
 
-// CreateRepo creates a repository
-func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
-	owner, err := models.GetUserByName(g.repoOwner)
-	if err != nil {
-		return err
-	}
-
-	var remoteAddr = repo.CloneURL
+func fullURL(opts base.MigrateOptions, remoteAddr string) (string, error) {
+	var fullRemoteAddr = remoteAddr
 	if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
-		u, err := url.Parse(repo.CloneURL)
+		u, err := url.Parse(remoteAddr)
 		if err != nil {
-			return err
+			return "", err
 		}
 		u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
 		if len(opts.AuthToken) > 0 {
 			u.User = url.UserPassword("oauth2", opts.AuthToken)
 		}
-		remoteAddr = u.String()
+		fullRemoteAddr = u.String()
+	}
+	return fullRemoteAddr, nil
+}
+
+// CreateRepo creates a repository
+func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
+	owner, err := models.GetUserByName(g.repoOwner)
+	if err != nil {
+		return err
 	}
 
+	remoteAddr, err := fullURL(opts, repo.CloneURL)
+	if err != nil {
+		return err
+	}
 	var r *models.Repository
 	if opts.MigrateToRepoID <= 0 {
 		r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{
@@ -224,7 +231,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
 }
 
 // CreateReleases creates releases
-func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases ...*base.Release) error {
+func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 	var rels = make([]*models.Release, 0, len(releases))
 	for _, release := range releases {
 		var rel = models.Release{
@@ -283,25 +290,27 @@ func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases
 
 			// download attachment
 			err = func() error {
+				// asset.DownloadURL maybe a local file
 				var rc io.ReadCloser
 				if asset.DownloadURL == nil {
-					rc, err = downloader.GetAsset(rel.TagName, rel.ID, asset.ID)
+					rc, err = asset.DownloadFunc()
 					if err != nil {
 						return err
 					}
 				} else {
-					resp, err := http.Get(*asset.DownloadURL)
+					rc, err = uri.Open(*asset.DownloadURL)
 					if err != nil {
 						return err
 					}
-					rc = resp.Body
 				}
+				defer rc.Close()
 				_, err = storage.Attachments.Save(attach.RelativePath(), rc)
 				return err
 			}()
 			if err != nil {
 				return err
 			}
+
 			rel.Attachments = append(rel.Attachments, &attach)
 		}
 
@@ -559,11 +568,12 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 
 	// download patch file
 	err := func() error {
-		resp, err := http.Get(pr.PatchURL)
+		// pr.PatchURL maybe a local file
+		ret, err := uri.Open(pr.PatchURL)
 		if err != nil {
 			return err
 		}
-		defer resp.Body.Close()
+		defer ret.Close()
 		pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
 		if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
 			return err
@@ -573,7 +583,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 			return err
 		}
 		defer f.Close()
-		_, err = io.Copy(f, resp.Body)
+		_, err = io.Copy(f, ret)
 		return err
 	}()
 	if err != nil {
@@ -859,3 +869,13 @@ func (g *GiteaLocalUploader) Rollback() error {
 	}
 	return nil
 }
+
+// Finish when migrating success, this will do some status update things.
+func (g *GiteaLocalUploader) Finish() error {
+	if g.repo == nil || g.repo.ID <= 0 {
+		return ErrRepoNotCreated
+	}
+
+	g.repo.Status = models.RepositoryReady
+	return models.UpdateRepositoryCols(g.repo, "status")
+}