diff options
author | zeripath <art27@cantab.net> | 2020-09-25 05:09:23 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-09-25 07:09:23 +0300 |
commit | 7a7f56044ae0b94d9f5e733d863821dbe222a395 (patch) | |
tree | 0a6ebe01ce4f48a117598c7b42e189a207314553 /modules/repository | |
parent | 6fa19a8458e993972be2355e0575a76823a75735 (diff) | |
download | gitea-7a7f56044ae0b94d9f5e733d863821dbe222a395.tar.gz gitea-7a7f56044ae0b94d9f5e733d863821dbe222a395.zip |
Adopt repositories (#12920)
* Don't automatically delete repository files if they are present
Prior to this PR Gitea would delete any repository files if they are
present during creation or migration. This can in certain circumstances
lead to data-loss and is slightly unpleasant.
This PR provides a mechanism for Gitea to adopt repositories on creation
and otherwise requires an explicit flag for deletion.
PushCreate is slightly different - the create will cause adoption if
that is allowed otherwise it will delete the data if that is allowed.
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Update swagger
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Fix tests and migrate overwrite
Signed-off-by: Andrew Thornton <art27@cantab.net>
* as per @lunny
Only offer to adopt or overwrite if the user can do that.
Allow the site administrator to adopt or overwrite in all
circumstances
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Use setting.Repository.DefaultBranch for the default branch
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Always set setting.Repository.DefaultBranch
Signed-off-by: Andrew Thornton <art27@cantab.net>
* update swagger
Signed-off-by: Andrew Thornton <art27@cantab.net>
* update templates
Signed-off-by: Andrew Thornton <art27@cantab.net>
* ensure repo closed
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Rewrite of adoption as per @6543 and @lunny
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Apply suggestions from code review
* update swagger
Signed-off-by: Andrew Thornton <art27@cantab.net>
* missing not
Signed-off-by: Andrew Thornton <art27@cantab.net>
* add modals and flash reporting
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Make the unadopted page searchable
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Add API
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Fix swagger
Signed-off-by: Andrew Thornton <art27@cantab.net>
* fix swagger
Signed-off-by: Andrew Thornton <art27@cantab.net>
* Handle empty and non-master branched repositories
Signed-off-by: Andrew Thornton <art27@cantab.net>
* placate lint
Signed-off-by: Andrew Thornton <art27@cantab.net>
* remove commented out code
Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
Diffstat (limited to 'modules/repository')
-rw-r--r-- | modules/repository/adopt.go | 272 | ||||
-rw-r--r-- | modules/repository/create.go | 73 | ||||
-rw-r--r-- | modules/repository/fork.go | 24 | ||||
-rw-r--r-- | modules/repository/generate.go | 14 | ||||
-rw-r--r-- | modules/repository/init.go | 89 |
5 files changed, 440 insertions, 32 deletions
diff --git a/modules/repository/adopt.go b/modules/repository/adopt.go new file mode 100644 index 0000000000..22cd6dd91f --- /dev/null +++ b/modules/repository/adopt.go @@ -0,0 +1,272 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repository + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + "github.com/gobwas/glob" + "github.com/unknwon/com" +) + +// AdoptRepository adopts a repository for the user/organization. +func AdoptRepository(doer, u *models.User, opts models.CreateRepoOptions) (*models.Repository, error) { + if !doer.IsAdmin && !u.CanCreateRepo() { + return nil, models.ErrReachLimitOfRepo{ + Limit: u.MaxRepoCreation, + } + } + + if len(opts.DefaultBranch) == 0 { + opts.DefaultBranch = setting.Repository.DefaultBranch + } + + repo := &models.Repository{ + OwnerID: u.ID, + Owner: u, + OwnerName: u.Name, + Name: opts.Name, + LowerName: strings.ToLower(opts.Name), + Description: opts.Description, + OriginalURL: opts.OriginalURL, + OriginalServiceType: opts.GitServiceType, + IsPrivate: opts.IsPrivate, + IsFsckEnabled: !opts.IsMirror, + CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, + Status: opts.Status, + IsEmpty: !opts.AutoInit, + } + + if err := models.WithTx(func(ctx models.DBContext) error { + repoPath := models.RepoPath(u.Name, repo.Name) + if !com.IsExist(repoPath) { + return models.ErrRepoNotExist{ + OwnerName: u.Name, + Name: repo.Name, + } + } + + if err := models.CreateRepository(ctx, doer, u, repo, true); err != nil { + return err + } + if err := adoptRepository(ctx, repoPath, doer, repo, opts); err != nil { + return fmt.Errorf("createDelegateHooks: %v", err) + } + + // Initialize Issue Labels if selected + if len(opts.IssueLabels) > 0 { + if err := models.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { + return fmt.Errorf("InitializeLabels: %v", err) + } + } + + if stdout, err := git.NewCommand("update-server-info"). + SetDescription(fmt.Sprintf("CreateRepository(git update-server-info): %s", repoPath)). + RunInDir(repoPath); err != nil { + log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return fmt.Errorf("CreateRepository(git update-server-info): %v", err) + } + return nil + }); err != nil { + return nil, err + } + + return repo, nil +} + +// DeleteUnadoptedRepository deletes unadopted repository files from the filesystem +func DeleteUnadoptedRepository(doer, u *models.User, repoName string) error { + if err := models.IsUsableRepoName(repoName); err != nil { + return err + } + + repoPath := models.RepoPath(u.Name, repoName) + if !com.IsExist(repoPath) { + return models.ErrRepoNotExist{ + OwnerName: u.Name, + Name: repoName, + } + } + + if exist, err := models.IsRepositoryExist(u, repoName); err != nil { + return err + } else if exist { + return models.ErrRepoAlreadyExist{ + Uname: u.Name, + Name: repoName, + } + } + + return util.RemoveAll(repoPath) +} + +// ListUnadoptedRepositories lists all the unadopted repositories that match the provided query +func ListUnadoptedRepositories(query string, opts *models.ListOptions) ([]string, int, error) { + globUser, _ := glob.Compile("*") + globRepo, _ := glob.Compile("*") + + qsplit := strings.SplitN(query, "/", 2) + if len(qsplit) > 0 && len(query) > 0 { + var err error + globUser, err = glob.Compile(qsplit[0]) + if err != nil { + log.Info("Invalid glob expresion '%s' (skipped): %v", qsplit[0], err) + } + if len(qsplit) > 1 { + globRepo, err = glob.Compile(qsplit[1]) + if err != nil { + log.Info("Invalid glob expresion '%s' (skipped): %v", qsplit[1], err) + } + } + } + start := (opts.Page - 1) * opts.PageSize + end := start + opts.PageSize + + repoNamesToCheck := make([]string, 0, opts.PageSize) + + repoNames := make([]string, 0, opts.PageSize) + var ctxUser *models.User + + count := 0 + + // We're going to iterate by pagesize. + root := filepath.Join(setting.RepoRootPath) + if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() || path == root { + return nil + } + + if !strings.ContainsRune(path[len(root)+1:], filepath.Separator) { + // Got a new user + + // Clean up old repoNamesToCheck + if len(repoNamesToCheck) > 0 { + repos, _, err := models.GetUserRepositories(&models.SearchRepoOptions{Actor: ctxUser, Private: true, ListOptions: models.ListOptions{ + Page: 1, + PageSize: opts.PageSize, + }, LowerNames: repoNamesToCheck}) + if err != nil { + return err + } + for _, name := range repoNamesToCheck { + found := false + repoLoopCatchup: + for i, repo := range repos { + if repo.LowerName == name { + found = true + repos = append(repos[:i], repos[i+1:]...) + break repoLoopCatchup + } + } + if !found { + if count >= start && count < end { + repoNames = append(repoNames, fmt.Sprintf("%s/%s", ctxUser.Name, name)) + } + count++ + } + } + repoNamesToCheck = repoNamesToCheck[:0] + } + + if !globUser.Match(info.Name()) { + return filepath.SkipDir + } + + ctxUser, err = models.GetUserByName(info.Name()) + if err != nil { + if models.IsErrUserNotExist(err) { + log.Debug("Missing user: %s", info.Name()) + return filepath.SkipDir + } + return err + } + return nil + } + + name := info.Name() + + if !strings.HasSuffix(name, ".git") { + return filepath.SkipDir + } + name = name[:len(name)-4] + if models.IsUsableRepoName(name) != nil || strings.ToLower(name) != name || !globRepo.Match(name) { + return filepath.SkipDir + } + if count < end { + repoNamesToCheck = append(repoNamesToCheck, name) + if len(repoNamesToCheck) >= opts.PageSize { + repos, _, err := models.GetUserRepositories(&models.SearchRepoOptions{Actor: ctxUser, Private: true, ListOptions: models.ListOptions{ + Page: 1, + PageSize: opts.PageSize, + }, LowerNames: repoNamesToCheck}) + if err != nil { + return err + } + for _, name := range repoNamesToCheck { + found := false + repoLoop: + for i, repo := range repos { + if repo.Name == name { + found = true + repos = append(repos[:i], repos[i+1:]...) + break repoLoop + } + } + if !found { + if count >= start && count < end { + repoNames = append(repoNames, fmt.Sprintf("%s/%s", ctxUser.Name, name)) + } + count++ + } + } + repoNamesToCheck = repoNamesToCheck[:0] + } + return filepath.SkipDir + } + count++ + return filepath.SkipDir + }); err != nil { + return nil, 0, err + } + + if len(repoNamesToCheck) > 0 { + repos, _, err := models.GetUserRepositories(&models.SearchRepoOptions{Actor: ctxUser, Private: true, ListOptions: models.ListOptions{ + Page: 1, + PageSize: opts.PageSize, + }, LowerNames: repoNamesToCheck}) + if err != nil { + return nil, 0, err + } + for _, name := range repoNamesToCheck { + found := false + repoLoop: + for i, repo := range repos { + if repo.LowerName == name { + found = true + repos = append(repos[:i], repos[i+1:]...) + break repoLoop + } + } + if !found { + if count >= start && count < end { + repoNames = append(repoNames, fmt.Sprintf("%s/%s", ctxUser.Name, name)) + } + count++ + } + } + } + return repoNames, count, nil +} diff --git a/modules/repository/create.go b/modules/repository/create.go index c180b9b948..e6a3e7081d 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -13,10 +13,12 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + + "github.com/unknwon/com" ) // CreateRepository creates a repository for the user/organization. -func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (_ *models.Repository, err error) { +func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (*models.Repository, error) { if !doer.IsAdmin && !u.CanCreateRepo() { return nil, models.ErrReachLimitOfRepo{ Limit: u.MaxRepoCreation, @@ -44,39 +46,64 @@ func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (_ *m TrustModel: opts.TrustModel, } - err = models.WithTx(func(ctx models.DBContext) error { - if err = models.CreateRepository(ctx, doer, u, repo); err != nil { + if err := models.WithTx(func(ctx models.DBContext) error { + if err := models.CreateRepository(ctx, doer, u, repo, false); err != nil { return err } // No need for init mirror. - if !opts.IsMirror { - repoPath := models.RepoPath(u.Name, repo.Name) - if err = initRepository(ctx, repoPath, doer, repo, opts); err != nil { - if err2 := util.RemoveAll(repoPath); err2 != nil { - log.Error("initRepository: %v", err) - return fmt.Errorf( - "delete repo directory %s/%s failed(2): %v", u.Name, repo.Name, err2) - } - return fmt.Errorf("initRepository: %v", err) + if opts.IsMirror { + return nil + } + + repoPath := models.RepoPath(u.Name, repo.Name) + if com.IsExist(repoPath) { + // repo already exists - We have two or three options. + // 1. We fail stating that the directory exists + // 2. We create the db repository to go with this data and adopt the git repo + // 3. We delete it and start afresh + // + // Previously Gitea would just delete and start afresh - this was naughty. + // So we will now fail and delegate to other functionality to adopt or delete + log.Error("Files already exist in %s and we are not going to adopt or delete.", repoPath) + return models.ErrRepoFilesAlreadyExist{ + Uname: u.Name, + Name: repo.Name, + } + } + + if err := initRepository(ctx, repoPath, doer, repo, opts); err != nil { + if err2 := util.RemoveAll(repoPath); err2 != nil { + log.Error("initRepository: %v", err) + return fmt.Errorf( + "delete repo directory %s/%s failed(2): %v", u.Name, repo.Name, err2) } + return fmt.Errorf("initRepository: %v", err) + } - // Initialize Issue Labels if selected - if len(opts.IssueLabels) > 0 { - if err = models.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { - return fmt.Errorf("InitializeLabels: %v", err) + // Initialize Issue Labels if selected + if len(opts.IssueLabels) > 0 { + if err := models.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { + if errDelete := models.DeleteRepository(doer, u.ID, repo.ID); errDelete != nil { + log.Error("Rollback deleteRepository: %v", errDelete) } + return fmt.Errorf("InitializeLabels: %v", err) } + } - if stdout, err := git.NewCommand("update-server-info"). - SetDescription(fmt.Sprintf("CreateRepository(git update-server-info): %s", repoPath)). - RunInDir(repoPath); err != nil { - log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - return fmt.Errorf("CreateRepository(git update-server-info): %v", err) + if stdout, err := git.NewCommand("update-server-info"). + SetDescription(fmt.Sprintf("CreateRepository(git update-server-info): %s", repoPath)). + RunInDir(repoPath); err != nil { + log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) + if errDelete := models.DeleteRepository(doer, u.ID, repo.ID); errDelete != nil { + log.Error("Rollback deleteRepository: %v", errDelete) } + return fmt.Errorf("CreateRepository(git update-server-info): %v", err) } return nil - }) + }); err != nil { + return nil, err + } - return repo, err + return repo, nil } diff --git a/modules/repository/fork.go b/modules/repository/fork.go index 169c391edd..cdd08e3d3c 100644 --- a/modules/repository/fork.go +++ b/modules/repository/fork.go @@ -46,11 +46,21 @@ func ForkRepository(doer, owner *models.User, oldRepo *models.Repository, name, oldRepoPath := oldRepo.RepoPath() err = models.WithTx(func(ctx models.DBContext) error { - if err = models.CreateRepository(ctx, doer, owner, repo); err != nil { + if err = models.CreateRepository(ctx, doer, owner, repo, false); err != nil { return err } + rollbackRemoveFn := func() { + if repo.ID == 0 { + return + } + if errDelete := models.DeleteRepository(doer, owner.ID, repo.ID); errDelete != nil { + log.Error("Rollback deleteRepository: %v", errDelete) + } + } + if err = models.IncrementRepoForkNum(ctx, oldRepo.ID); err != nil { + rollbackRemoveFn() return err } @@ -60,6 +70,7 @@ func ForkRepository(doer, owner *models.User, oldRepo *models.Repository, name, SetDescription(fmt.Sprintf("ForkRepository(git clone): %s to %s", oldRepo.FullName(), repo.FullName())). RunInDirTimeout(10*time.Minute, ""); err != nil { log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, oldRepo, stdout, err) + rollbackRemoveFn() return fmt.Errorf("git clone: %v", err) } @@ -67,10 +78,12 @@ func ForkRepository(doer, owner *models.User, oldRepo *models.Repository, name, SetDescription(fmt.Sprintf("ForkRepository(git update-server-info): %s", repo.FullName())). RunInDir(repoPath); err != nil { log.Error("Fork Repository (git update-server-info) failed for %v:\nStdout: %s\nError: %v", repo, stdout, err) + rollbackRemoveFn() return fmt.Errorf("git update-server-info: %v", err) } if err = createDelegateHooks(repoPath); err != nil { + rollbackRemoveFn() return fmt.Errorf("createDelegateHooks: %v", err) } return nil @@ -86,5 +99,12 @@ func ForkRepository(doer, owner *models.User, oldRepo *models.Repository, name, if err := models.CopyLanguageStat(oldRepo, repo); err != nil { log.Error("Copy language stat from oldRepo failed") } - return repo, models.CopyLFS(ctx, repo, oldRepo) + + if err := models.CopyLFS(ctx, repo, oldRepo); err != nil { + if errDelete := models.DeleteRepository(doer, owner.ID, repo.ID); errDelete != nil { + log.Error("Rollback deleteRepository: %v", errDelete) + } + return nil, err + } + return repo, nil } diff --git a/modules/repository/generate.go b/modules/repository/generate.go index 1314464a6e..5d1ef72b6c 100644 --- a/modules/repository/generate.go +++ b/modules/repository/generate.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/util" "github.com/huandu/xstrings" + "github.com/unknwon/com" ) type transformer struct { @@ -246,12 +247,19 @@ func GenerateRepository(ctx models.DBContext, doer, owner *models.User, template TrustModel: templateRepo.TrustModel, } - if err = models.CreateRepository(ctx, doer, owner, generateRepo); err != nil { + if err = models.CreateRepository(ctx, doer, owner, generateRepo, false); err != nil { return nil, err } - repoPath := models.RepoPath(owner.Name, generateRepo.Name) - if err = checkInitRepository(repoPath); err != nil { + repoPath := generateRepo.RepoPath() + if com.IsExist(repoPath) { + return nil, models.ErrRepoFilesAlreadyExist{ + Uname: generateRepo.OwnerName, + Name: generateRepo.Name, + } + } + + if err = checkInitRepository(owner.Name, generateRepo.Name); err != nil { return generateRepo, err } diff --git a/modules/repository/init.go b/modules/repository/init.go index d066544a85..707f8f5250 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -172,10 +172,14 @@ func initRepoCommit(tmpPath string, repo *models.Repository, u *models.User, def return nil } -func checkInitRepository(repoPath string) (err error) { +func checkInitRepository(owner, name string) (err error) { // Somehow the directory could exist. + repoPath := models.RepoPath(owner, name) if com.IsExist(repoPath) { - return fmt.Errorf("checkInitRepository: path already exists: %s", repoPath) + return models.ErrRepoFilesAlreadyExist{ + Uname: owner, + Name: name, + } } // Init git bare new repository. @@ -187,9 +191,85 @@ func checkInitRepository(repoPath string) (err error) { return nil } +func adoptRepository(ctx models.DBContext, repoPath string, u *models.User, repo *models.Repository, opts models.CreateRepoOptions) (err error) { + if !com.IsExist(repoPath) { + return fmt.Errorf("adoptRepository: path does not already exist: %s", repoPath) + } + + if err := createDelegateHooks(repoPath); err != nil { + return fmt.Errorf("createDelegateHooks: %v", err) + } + + // Re-fetch the repository from database before updating it (else it would + // override changes that were done earlier with sql) + if repo, err = models.GetRepositoryByIDCtx(ctx, repo.ID); err != nil { + return fmt.Errorf("getRepositoryByID: %v", err) + } + + repo.IsEmpty = false + gitRepo, err := git.OpenRepository(repo.RepoPath()) + if err != nil { + return fmt.Errorf("openRepository: %v", err) + } + defer gitRepo.Close() + if len(opts.DefaultBranch) > 0 { + repo.DefaultBranch = opts.DefaultBranch + + if err = gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { + return fmt.Errorf("setDefaultBranch: %v", err) + } + } else { + repo.DefaultBranch, err = gitRepo.GetDefaultBranch() + if err != nil { + repo.DefaultBranch = setting.Repository.DefaultBranch + if err = gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { + return fmt.Errorf("setDefaultBranch: %v", err) + } + } + + repo.DefaultBranch = strings.TrimPrefix(repo.DefaultBranch, git.BranchPrefix) + } + branches, _ := gitRepo.GetBranches() + found := false + hasDefault := false + hasMaster := false + for _, branch := range branches { + if branch == repo.DefaultBranch { + found = true + break + } else if branch == setting.Repository.DefaultBranch { + hasDefault = true + } else if branch == "master" { + hasMaster = true + } + } + if !found { + if hasDefault { + repo.DefaultBranch = setting.Repository.DefaultBranch + } else if hasMaster { + repo.DefaultBranch = "master" + } else if len(branches) > 0 { + repo.DefaultBranch = branches[0] + } else { + repo.IsEmpty = true + repo.DefaultBranch = setting.Repository.DefaultBranch + } + + if err = gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { + return fmt.Errorf("setDefaultBranch: %v", err) + } + } + + if err = models.UpdateRepositoryCtx(ctx, repo, false); err != nil { + return fmt.Errorf("updateRepository: %v", err) + } + + return nil +} + // InitRepository initializes README and .gitignore if needed. func initRepository(ctx models.DBContext, repoPath string, u *models.User, repo *models.Repository, opts models.CreateRepoOptions) (err error) { - if err = checkInitRepository(repoPath); err != nil { + if err = checkInitRepository(repo.OwnerName, repo.Name); err != nil { return err } @@ -225,7 +305,8 @@ func initRepository(ctx models.DBContext, repoPath string, u *models.User, repo repo.IsEmpty = true } - repo.DefaultBranch = "master" + repo.DefaultBranch = setting.Repository.DefaultBranch + if len(opts.DefaultBranch) > 0 { repo.DefaultBranch = opts.DefaultBranch gitRepo, err := git.OpenRepository(repo.RepoPath()) |