diff options
Diffstat (limited to 'services/repository')
50 files changed, 3637 insertions, 1764 deletions
diff --git a/services/repository/adopt.go b/services/repository/adopt.go index e37909e7ab..2bd1c55de4 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -16,7 +16,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" @@ -28,18 +27,30 @@ import ( "github.com/gobwas/glob" ) +func deleteFailedAdoptRepository(repoID int64) error { + return db.WithTx(db.DefaultContext, func(ctx context.Context) error { + if err := deleteDBRepository(ctx, repoID); err != nil { + return fmt.Errorf("deleteDBRepository: %w", err) + } + if err := git_model.DeleteRepoBranches(ctx, repoID); err != nil { + return fmt.Errorf("deleteRepoBranches: %w", err) + } + return repo_model.DeleteRepoReleases(ctx, repoID) + }) +} + // AdoptRepository adopts pre-existing repository files for the user/organization. -func AdoptRepository(ctx context.Context, doer, u *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { - if !doer.IsAdmin && !u.CanCreateRepo() { +func AdoptRepository(ctx context.Context, doer, owner *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { + if !doer.CanCreateRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ - Limit: u.MaxRepoCreation, + Limit: owner.MaxRepoCreation, } } repo := &repo_model.Repository{ - OwnerID: u.ID, - Owner: u, - OwnerName: u.Name, + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, Name: opts.Name, LowerName: strings.ToLower(opts.Name), Description: opts.Description, @@ -48,75 +59,67 @@ func AdoptRepository(ctx context.Context, doer, u *user_model.User, opts CreateR IsPrivate: opts.IsPrivate, IsFsckEnabled: !opts.IsMirror, CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, - Status: opts.Status, + Status: repo_model.RepositoryBeingMigrated, IsEmpty: !opts.AutoInit, } - if err := db.WithTx(ctx, func(ctx context.Context) error { - repoPath := repo_model.RepoPath(u.Name, repo.Name) - isExist, err := util.IsExist(repoPath) + // 1 - create the repository database operations first + err := db.WithTx(ctx, func(ctx context.Context) error { + return createRepositoryInDB(ctx, doer, owner, repo, false) + }) + if err != nil { + return nil, err + } + + // last - clean up if something goes wrong + // WARNING: Don't override all later err with local variables + defer func() { if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return err - } - if !isExist { - return repo_model.ErrRepoNotExist{ - OwnerName: u.Name, - Name: repo.Name, + // we can not use the ctx because it maybe canceled or timeout + if errDel := deleteFailedAdoptRepository(repo.ID); errDel != nil { + log.Error("Failed to delete repository %s that could not be adopted: %v", repo.FullName(), errDel) } } + }() - if err := CreateRepositoryByExample(ctx, doer, u, repo, true, false); err != nil { - return err - } - - // Re-fetch the repository from database before updating it (else it would - // override changes that were done earlier with sql) - if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { - return fmt.Errorf("getRepositoryByID: %w", err) - } - - if err := adoptRepository(ctx, repoPath, repo, opts.DefaultBranch); err != nil { - return fmt.Errorf("adoptRepository: %w", err) - } + // Re-fetch the repository from database before updating it (else it would + // override changes that were done earlier with sql) + if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { + return nil, fmt.Errorf("getRepositoryByID: %w", err) + } - if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { - return fmt.Errorf("checkDaemonExportOK: %w", err) - } + // 2 - adopt the repository from disk + if err = adoptRepository(ctx, repo, opts.DefaultBranch); err != nil { + return nil, fmt.Errorf("adoptRepository: %w", err) + } - // Initialize Issue Labels if selected - if len(opts.IssueLabels) > 0 { - if err := repo_module.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { - return fmt.Errorf("InitializeLabels: %w", err) - } - } + // 3 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } - if stdout, _, err := git.NewCommand(ctx, "update-server-info"). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - return fmt.Errorf("CreateRepository(git update-server-info): %w", err) - } - return nil - }); err != nil { - return nil, err + // 4 - update repository status + repo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } - notify_service.AdoptRepository(ctx, doer, u, repo) + notify_service.AdoptRepository(ctx, doer, owner, repo) return repo, nil } -func adoptRepository(ctx context.Context, repoPath string, repo *repo_model.Repository, defaultBranch string) (err error) { - isExist, err := util.IsExist(repoPath) +func adoptRepository(ctx context.Context, repo *repo_model.Repository, defaultBranch string) (err error) { + isExist, err := gitrepo.IsRepositoryExist(ctx, repo) if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) + log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) return err } if !isExist { - return fmt.Errorf("adoptRepository: path does not already exist: %s", repoPath) + return fmt.Errorf("adoptRepository: path does not already exist: %s", repo.FullName()) } - if err := repo_module.CreateDelegateHooks(repoPath); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo); err != nil { return fmt.Errorf("createDelegateHooks: %w", err) } @@ -193,8 +196,13 @@ func adoptRepository(ctx context.Context, repoPath string, repo *repo_model.Repo return fmt.Errorf("setDefaultBranch: %w", err) } } - if err = repo_module.UpdateRepository(ctx, repo, false); err != nil { - return fmt.Errorf("updateRepository: %w", err) + + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_empty", "default_branch"); err != nil { + return fmt.Errorf("UpdateRepositoryCols: %w", err) + } + + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) } return nil @@ -257,7 +265,7 @@ func checkUnadoptedRepositories(ctx context.Context, userName string, repoNamesT } return err } - repos, _, err := repo_model.GetUserRepositories(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.GetUserRepositories(ctx, repo_model.SearchRepoOptions{ Actor: ctxUser, Private: true, ListOptions: db.ListOptions{ diff --git a/services/repository/adopt_test.go b/services/repository/adopt_test.go index 123cedc1f2..86f586c748 100644 --- a/services/repository/adopt_test.go +++ b/services/repository/adopt_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) @@ -28,7 +29,7 @@ func TestCheckUnadoptedRepositories_Add(t *testing.T) { } total := 30 - for i := 0; i < total; i++ { + for range total { unadopted.add("something") } @@ -71,7 +72,7 @@ func TestListUnadoptedRepositories_ListOptions(t *testing.T) { username := "user2" unadoptedList := []string{path.Join(username, "unadopted1"), path.Join(username, "unadopted2")} for _, unadopted := range unadoptedList { - _ = os.Mkdir(path.Join(setting.RepoRootPath, unadopted+".git"), 0o755) + _ = os.Mkdir(filepath.Join(setting.RepoRootPath, unadopted+".git"), 0o755) } opts := db.ListOptions{Page: 1, PageSize: 1} @@ -89,10 +90,36 @@ func TestListUnadoptedRepositories_ListOptions(t *testing.T) { func TestAdoptRepository(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - assert.NoError(t, unittest.SyncDirs(filepath.Join(setting.RepoRootPath, "user2", "repo1.git"), filepath.Join(setting.RepoRootPath, "user2", "test-adopt.git"))) + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - _, err := AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"}) + + // a successful adopt + destDir := filepath.Join(setting.RepoRootPath, user2.Name, "test-adopt.git") + assert.NoError(t, unittest.SyncDirs(filepath.Join(setting.RepoRootPath, user2.Name, "repo1.git"), destDir)) + + adoptedRepo, err := AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"}) assert.NoError(t, err) repoTestAdopt := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: "test-adopt"}) assert.Equal(t, "sha1", repoTestAdopt.ObjectFormatName) + + // just delete the adopted repo's db records + err = deleteFailedAdoptRepository(adoptedRepo.ID) + assert.NoError(t, err) + + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: "test-adopt"}) + + // a failed adopt because some mock data + // remove the hooks directory and create a file so that we cannot create the hooks successfully + _ = os.RemoveAll(filepath.Join(destDir, "hooks", "update.d")) + assert.NoError(t, os.WriteFile(filepath.Join(destDir, "hooks", "update.d"), []byte("tests"), os.ModePerm)) + + adoptedRepo, err = AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"}) + assert.Error(t, err) + assert.Nil(t, adoptedRepo) + + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: "test-adopt"}) + + exist, err := util.IsExist(repo_model.RepoPath(user2.Name, "test-adopt")) + assert.NoError(t, err) + assert.True(t, exist) // the repository should be still in the disk } diff --git a/services/repository/archiver/archiver.go b/services/repository/archiver/archiver.go index d39acc080d..a657e3884c 100644 --- a/services/repository/archiver/archiver.go +++ b/services/repository/archiver/archiver.go @@ -44,7 +44,7 @@ type ErrUnknownArchiveFormat struct { // Error implements error func (err ErrUnknownArchiveFormat) Error() string { - return fmt.Sprintf("unknown format: %s", err.RequestNameType) + return "unknown format: " + err.RequestNameType } // Is implements error @@ -60,7 +60,7 @@ type RepoRefNotFoundError struct { // Error implements error. func (e RepoRefNotFoundError) Error() string { - return fmt.Sprintf("unrecognized repository reference: %s", e.RefShortName) + return "unrecognized repository reference: " + e.RefShortName } func (e RepoRefNotFoundError) Is(err error) bool { diff --git a/services/repository/archiver/archiver_test.go b/services/repository/archiver/archiver_test.go index 522f90558a..87324ad38c 100644 --- a/services/repository/archiver/archiver_test.go +++ b/services/repository/archiver/archiver_test.go @@ -33,7 +33,7 @@ func TestArchive_Basic(t *testing.T) { bogusReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) - assert.EqualValues(t, firstCommit+".zip", bogusReq.GetArchiveName()) + assert.Equal(t, firstCommit+".zip", bogusReq.GetArchiveName()) // Check a series of bogus requests. // Step 1, valid commit with a bad extension. @@ -54,12 +54,12 @@ func TestArchive_Basic(t *testing.T) { bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "master.zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) - assert.EqualValues(t, "master.zip", bogusReq.GetArchiveName()) + assert.Equal(t, "master.zip", bogusReq.GetArchiveName()) bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "test/archive.zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) - assert.EqualValues(t, "test-archive.zip", bogusReq.GetArchiveName()) + assert.Equal(t, "test-archive.zip", bogusReq.GetArchiveName()) // Now two valid requests, firstCommit with valid extensions. zipReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") diff --git a/services/repository/avatar.go b/services/repository/avatar.go index 15e51d4a25..26bf6da465 100644 --- a/services/repository/avatar.go +++ b/services/repository/avatar.go @@ -40,7 +40,7 @@ func UploadAvatar(ctx context.Context, repo *repo_model.Repository, data []byte) // Users can upload the same image to other repo - prefix it with ID // Then repo will be removed - only it avatar file will be removed repo.Avatar = newAvatar - if err := repo_model.UpdateRepositoryCols(ctx, repo, "avatar"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "avatar"); err != nil { return fmt.Errorf("UploadAvatar: Update repository avatar: %w", err) } @@ -77,7 +77,7 @@ func DeleteAvatar(ctx context.Context, repo *repo_model.Repository) error { defer committer.Close() repo.Avatar = "" - if err := repo_model.UpdateRepositoryCols(ctx, repo, "avatar"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "avatar"); err != nil { return fmt.Errorf("DeleteAvatar: Update repository avatar: %w", err) } @@ -112,5 +112,5 @@ func generateAvatar(ctx context.Context, templateRepo, generateRepo *repo_model. return err } - return repo_model.UpdateRepositoryCols(ctx, generateRepo, "avatar") + return repo_model.UpdateRepositoryColsNoAutoTime(ctx, generateRepo, "avatar") } diff --git a/services/repository/avatar_test.go b/services/repository/avatar_test.go index bea820e85f..2dc5173eec 100644 --- a/services/repository/avatar_test.go +++ b/services/repository/avatar_test.go @@ -59,7 +59,7 @@ func TestDeleteAvatar(t *testing.T) { err = DeleteAvatar(db.DefaultContext, repo) assert.NoError(t, err) - assert.Equal(t, "", repo.Avatar) + assert.Empty(t, repo.Avatar) } func TestGenerateAvatar(t *testing.T) { diff --git a/services/repository/branch.go b/services/repository/branch.go index 08c53bbb6a..dd00ca7dcd 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -30,6 +30,7 @@ import ( "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" webhook_module "code.gitea.io/gitea/modules/webhook" + actions_service "code.gitea.io/gitea/services/actions" notify_service "code.gitea.io/gitea/services/notify" release_service "code.gitea.io/gitea/services/release" files_service "code.gitea.io/gitea/services/repository/files" @@ -302,7 +303,7 @@ func SyncBranchesToDB(ctx context.Context, repoID, pusherID int64, branchNames, // For other batches, it will hit optimization 4. if len(branchNames) != len(commitIDs) { - return fmt.Errorf("branchNames and commitIDs length not match") + return errors.New("branchNames and commitIDs length not match") } return db.WithTx(ctx, func(ctx context.Context) error { @@ -409,11 +410,11 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, doer *user_m return "target_exist", nil } - if gitRepo.IsBranchExist(to) { + if gitrepo.IsBranchExist(ctx, repo, to) { return "target_exist", nil } - if !gitRepo.IsBranchExist(from) { + if !gitrepo.IsBranchExist(ctx, repo, from) { return "from_not_exist", nil } @@ -452,7 +453,7 @@ func RenameBranch(ctx context.Context, repo *repo_model.Repository, doer *user_m log.Error("DeleteCronTaskByRepo: %v", err) } // cancel running cron jobs of this repository and delete old schedules - if err := actions_model.CancelPreviousJobs( + if err := actions_service.CancelPreviousJobs( ctx, repo.ID, from, @@ -617,12 +618,12 @@ func AddAllRepoBranchesToSyncQueue(ctx context.Context) error { return nil } -func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, newBranchName string) error { +func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newBranchName string) error { if repo.DefaultBranch == newBranchName { return nil } - if !gitRepo.IsBranchExist(newBranchName) { + if !gitrepo.IsBranchExist(ctx, repo, newBranchName) { return git_model.ErrBranchNotExist{ BranchName: newBranchName, } @@ -639,7 +640,7 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitR log.Error("DeleteCronTaskByRepo: %v", err) } // cancel running cron jobs of this repository and delete old schedules - if err := actions_model.CancelPreviousJobs( + if err := actions_service.CancelPreviousJobs( ctx, repo.ID, oldDefaultBranchName, @@ -662,15 +663,23 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitR } } + // clear divergence cache + if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil { + log.Error("DelRepoDivergenceFromCache: %v", err) + } + notify_service.ChangeDefaultBranch(ctx, repo) return nil } // BranchDivergingInfo contains the information about the divergence of a head branch to the base branch. -// This struct is also used in templates, so it needs to search for all references before changing it. type BranchDivergingInfo struct { + // whether the base branch contains new commits which are not in the head branch BaseHasNewCommits bool + + // behind/after are number of commits that the head branch is behind/after the base branch, it's 0 if it's unable to calculate. + // there could be a case that BaseHasNewCommits=true while the behind/after are both 0 (unable to calculate). HeadCommitsBehind int HeadCommitsAhead int } @@ -681,11 +690,20 @@ func GetBranchDivergingInfo(ctx reqctx.RequestContext, baseRepo *repo_model.Repo if err != nil { return nil, err } - + if headGitBranch.IsDeleted { + return nil, git_model.ErrBranchNotExist{ + BranchName: headBranch, + } + } baseGitBranch, err := git_model.GetBranch(ctx, baseRepo.ID, baseBranch) if err != nil { return nil, err } + if baseGitBranch.IsDeleted { + return nil, git_model.ErrBranchNotExist{ + BranchName: baseBranch, + } + } info := &BranchDivergingInfo{} if headGitBranch.CommitID == baseGitBranch.CommitID { @@ -720,5 +738,6 @@ func GetBranchDivergingInfo(ctx reqctx.RequestContext, baseRepo *repo_model.Repo } info.HeadCommitsBehind, info.HeadCommitsAhead = diff.Behind, diff.Ahead + info.BaseHasNewCommits = info.HeadCommitsBehind > 0 return info, nil } diff --git a/services/repository/check.go b/services/repository/check.go index acca15daf2..ffcd5ac749 100644 --- a/services/repository/check.go +++ b/services/repository/check.go @@ -86,10 +86,10 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args git.TrustedCmdA // GitGcRepo calls 'git gc' to remove unnecessary files and optimize the local repository func GitGcRepo(ctx context.Context, repo *repo_model.Repository, timeout time.Duration, args git.TrustedCmdArgs) error { log.Trace("Running git gc on %-v", repo) - command := git.NewCommand(ctx, "gc").AddArguments(args...) + command := git.NewCommand("gc").AddArguments(args...) var stdout string var err error - stdout, _, err = command.RunStdString(&git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()}) + stdout, _, err = command.RunStdString(ctx, &git.RunOpts{Timeout: timeout, Dir: repo.RepoPath()}) if err != nil { log.Error("Repository garbage collection failed for %-v. Stdout: %s\nError: %v", repo, stdout, err) desc := fmt.Sprintf("Repository garbage collection failed for %s. Stdout: %s\nError: %v", repo.RepoPath(), stdout, err) @@ -162,7 +162,7 @@ func DeleteMissingRepositories(ctx context.Context, doer *user_model.User) error default: } log.Trace("Deleting %d/%d...", repo.OwnerID, repo.ID) - if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { + if err := DeleteRepositoryDirectly(ctx, repo.ID); err != nil { log.Error("Failed to DeleteRepository %-v: Error: %v", repo, err) if err2 := system_model.CreateRepositoryNotice("Failed to DeleteRepository %s [%d]: Error: %v", repo.FullName(), repo.ID, err); err2 != nil { log.Error("CreateRepositoryNotice: %v", err) diff --git a/services/repository/commitstatus/commitstatus.go b/services/repository/commitstatus/commitstatus.go index f369a303e6..fa7a89882a 100644 --- a/services/repository/commitstatus/commitstatus.go +++ b/services/repository/commitstatus/commitstatus.go @@ -14,17 +14,17 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/commitstatus" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" - api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/services/notify" ) func getCacheKey(repoID int64, brancheName string) string { - hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%d:%s", repoID, brancheName))) + hashBytes := sha256.Sum256(fmt.Appendf(nil, "%d:%s", repoID, brancheName)) return fmt.Sprintf("commit_status:%x", hashBytes) } @@ -47,7 +47,7 @@ func getCommitStatusCache(repoID int64, branchName string) *commitStatusCacheVal return nil } -func updateCommitStatusCache(repoID int64, branchName string, state api.CommitStatusState, targetURL string) error { +func updateCommitStatusCache(repoID int64, branchName string, state commitstatus.CommitStatusState, targetURL string) error { c := cache.GetCache() bs, err := json.Marshal(commitStatusCacheValue{ State: state.String(), @@ -127,7 +127,7 @@ func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Rep for i, repo := range repos { if cv := getCommitStatusCache(repo.ID, repo.DefaultBranch); cv != nil { results[i] = &git_model.CommitStatus{ - State: api.CommitStatusState(cv.State), + State: commitstatus.CommitStatusState(cv.State), TargetURL: cv.TargetURL, } } else { diff --git a/services/repository/contributors_graph.go b/services/repository/contributors_graph.go index b0748f8ee3..a4ae505313 100644 --- a/services/repository/contributors_graph.go +++ b/services/repository/contributors_graph.go @@ -125,13 +125,13 @@ func getExtendedCommitStats(repo *git.Repository, revision string /*, limit int _ = stdoutWriter.Close() }() - gitCmd := git.NewCommand(repo.Ctx, "log", "--shortstat", "--no-merges", "--pretty=format:---%n%aN%n%aE%n%as", "--reverse") + gitCmd := git.NewCommand("log", "--shortstat", "--no-merges", "--pretty=format:---%n%aN%n%aE%n%as", "--reverse") // AddOptionFormat("--max-count=%d", limit) gitCmd.AddDynamicArguments(baseCommit.ID.String()) var extendedCommitStats []*ExtendedCommitStats stderr := new(strings.Builder) - err = gitCmd.Run(&git.RunOpts{ + err = gitCmd.Run(repo.Ctx, &git.RunOpts{ Dir: repo.Path, Stdout: stdoutWriter, Stderr: stderr, diff --git a/services/repository/contributors_graph_test.go b/services/repository/contributors_graph_test.go index 6db93f6a64..7d32b1c931 100644 --- a/services/repository/contributors_graph_test.go +++ b/services/repository/contributors_graph_test.go @@ -38,14 +38,14 @@ func TestRepository_ContributorsGraph(t *testing.T) { keys = append(keys, k) } slices.Sort(keys) - assert.EqualValues(t, []string{ + assert.Equal(t, []string{ "ethantkoenig@gmail.com", "jimmy.praet@telenet.be", "jon@allspice.io", "total", // generated summary }, keys) - assert.EqualValues(t, &ContributorData{ + assert.Equal(t, &ContributorData{ Name: "Ethan Koenig", AvatarLink: "/assets/img/avatar_default.png", TotalCommits: 1, @@ -58,7 +58,7 @@ func TestRepository_ContributorsGraph(t *testing.T) { }, }, }, data["ethantkoenig@gmail.com"]) - assert.EqualValues(t, &ContributorData{ + assert.Equal(t, &ContributorData{ Name: "Total", AvatarLink: "", TotalCommits: 3, diff --git a/services/repository/create.go b/services/repository/create.go index 23aacd6f95..bed02e5d7e 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/models/perm" access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" + system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" @@ -28,7 +29,6 @@ import ( "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/templates/vars" - "code.gitea.io/gitea/modules/util" ) // CreateRepoOptions contains the create repository options @@ -52,7 +52,7 @@ type CreateRepoOptions struct { ObjectFormatName string } -func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, repoPath string, opts CreateRepoOptions) error { +func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir string, opts CreateRepoOptions) error { commitTimeStr := time.Now().Format(time.RFC3339) authorSig := repo.Owner.NewGitSig() @@ -67,8 +67,8 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, ) // Clone to temporary path and do the init commit. - if stdout, _, err := git.NewCommand(ctx, "clone").AddDynamicArguments(repoPath, tmpDir). - RunStdString(&git.RunOpts{Dir: "", Env: env}); err != nil { + if stdout, _, err := git.NewCommand("clone").AddDynamicArguments(repo.RepoPath(), tmpDir). + RunStdString(ctx, &git.RunOpts{Dir: "", Env: env}); err != nil { log.Error("Failed to clone from %v into %s: stdout: %s\nError: %v", repo, tmpDir, stdout, err) return fmt.Errorf("git clone: %w", err) } @@ -100,8 +100,8 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, // .gitignore if len(opts.Gitignores) > 0 { var buf bytes.Buffer - names := strings.Split(opts.Gitignores, ",") - for _, name := range names { + names := strings.SplitSeq(opts.Gitignores, ",") + for name := range names { data, err = options.Gitignore(name) if err != nil { return fmt.Errorf("GetRepoInitFile[%s]: %w", name, err) @@ -139,24 +139,23 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, } // InitRepository initializes README and .gitignore if needed. -func initRepository(ctx context.Context, repoPath string, u *user_model.User, repo *repo_model.Repository, opts CreateRepoOptions) (err error) { - if err = repo_module.CheckInitRepository(ctx, repo.OwnerName, repo.Name, opts.ObjectFormatName); err != nil { - return err +func initRepository(ctx context.Context, u *user_model.User, repo *repo_model.Repository, opts CreateRepoOptions) (err error) { + // Init git bare new repository. + if err = git.InitRepository(ctx, repo.RepoPath(), true, repo.ObjectFormatName); err != nil { + return fmt.Errorf("git.InitRepository: %w", err) + } else if err = gitrepo.CreateDelegateHooks(ctx, repo); err != nil { + return fmt.Errorf("createDelegateHooks: %w", err) } // Initialize repository according to user's choice. if opts.AutoInit { - tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name) + tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("repos-" + repo.Name) if err != nil { - return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.RepoPath(), err) + return fmt.Errorf("failed to create temp dir for repository %s: %w", repo.FullName(), err) } - defer func() { - if err := util.RemoveAll(tmpDir); err != nil { - log.Warn("Unable to remove temporary directory: %s: Error: %v", tmpDir, err) - } - }() + defer cleanup() - if err = prepareRepoCommit(ctx, repo, tmpDir, repoPath, opts); err != nil { + if err = prepareRepoCommit(ctx, repo, tmpDir, opts); err != nil { return fmt.Errorf("prepareRepoCommit: %w", err) } @@ -192,18 +191,25 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re } } - if err = UpdateRepository(ctx, repo, false); err != nil { + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_empty", "default_branch", "default_wiki_branch"); err != nil { return fmt.Errorf("updateRepository: %w", err) } + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + return nil } // CreateRepositoryDirectly creates a repository for the user/organization. -func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { - if !doer.IsAdmin && !u.CanCreateRepo() { +// if needsUpdateToReady is true, it will update the repository status to ready when success +func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User, + opts CreateRepoOptions, needsUpdateToReady bool, +) (*repo_model.Repository, error) { + if !doer.CanCreateRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ - Limit: u.MaxRepoCreation, + Limit: owner.MaxRepoCreation, } } @@ -223,9 +229,9 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt } repo := &repo_model.Repository{ - OwnerID: u.ID, - Owner: u, - OwnerName: u.Name, + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, Name: opts.Name, LowerName: strings.ToLower(opts.Name), Description: opts.Description, @@ -244,101 +250,91 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt ObjectFormatName: opts.ObjectFormatName, } - var rollbackRepo *repo_model.Repository - - if err := db.WithTx(ctx, func(ctx context.Context) error { - if err := CreateRepositoryByExample(ctx, doer, u, repo, false, false); err != nil { - return err - } - - // No need for init mirror. - if opts.IsMirror { - return nil - } + // 1 - create the repository database operations first + err := db.WithTx(ctx, func(ctx context.Context) error { + return createRepositoryInDB(ctx, doer, owner, repo, false) + }) + if err != nil { + return nil, err + } - repoPath := repo_model.RepoPath(u.Name, repo.Name) - isExist, err := util.IsExist(repoPath) + // last - clean up if something goes wrong + // WARNING: Don't override all later err with local variables + defer func() { if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return err - } - if isExist { - // repo already exists - We have two or three options. - // 1. We fail stating that the directory exists - // 2. We create the db repository to go with this data and adopt the git repo - // 3. We delete it and start afresh - // - // Previously Gitea would just delete and start afresh - this was naughty. - // So we will now fail and delegate to other functionality to adopt or delete - log.Error("Files already exist in %s and we are not going to adopt or delete.", repoPath) - return repo_model.ErrRepoFilesAlreadyExist{ - Uname: u.Name, - Name: repo.Name, - } + // we can not use the ctx because it maybe canceled or timeout + cleanupRepository(repo.ID) } + }() - if err = initRepository(ctx, repoPath, doer, repo, opts); err != nil { - if err2 := util.RemoveAll(repoPath); err2 != nil { - log.Error("initRepository: %v", err) - return fmt.Errorf( - "delete repo directory %s/%s failed(2): %v", u.Name, repo.Name, err2) - } - return fmt.Errorf("initRepository: %w", err) - } + // No need for init mirror. + if opts.IsMirror { + return repo, nil + } - // Initialize Issue Labels if selected - if len(opts.IssueLabels) > 0 { - if err = repo_module.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { - rollbackRepo = repo - rollbackRepo.OwnerID = u.ID - return fmt.Errorf("InitializeLabels: %w", err) - } + // 2 - check whether the repository with the same storage exists + var isExist bool + isExist, err = gitrepo.IsRepositoryExist(ctx, repo) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) + return nil, err + } + if isExist { + log.Error("Files already exist in %s and we are not going to adopt or delete.", repo.FullName()) + // Don't return directly, we need err in defer to cleanupRepository + err = repo_model.ErrRepoFilesAlreadyExist{ + Uname: repo.OwnerName, + Name: repo.Name, } + return nil, err + } - if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { - return fmt.Errorf("checkDaemonExportOK: %w", err) - } + // 3 - init git repository in storage + if err = initRepository(ctx, doer, repo, opts); err != nil { + return nil, fmt.Errorf("initRepository: %w", err) + } - if stdout, _, err := git.NewCommand(ctx, "update-server-info"). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - rollbackRepo = repo - rollbackRepo.OwnerID = u.ID - return fmt.Errorf("CreateRepository(git update-server-info): %w", err) + // 4 - Initialize Issue Labels if selected + if len(opts.IssueLabels) > 0 { + if err = repo_module.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { + return nil, fmt.Errorf("InitializeLabels: %w", err) } + } - // update licenses - var licenses []string - if len(opts.License) > 0 { - licenses = append(licenses, ConvertLicenseName(opts.License)) + // 5 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } - stdout, _, err := git.NewCommand(ctx, "rev-parse", "HEAD").RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { - log.Error("CreateRepository(git rev-parse HEAD) in %v: Stdout: %s\nError: %v", repo, stdout, err) - rollbackRepo = repo - rollbackRepo.OwnerID = u.ID - return fmt.Errorf("CreateRepository(git rev-parse HEAD): %w", err) - } - if err := repo_model.UpdateRepoLicenses(ctx, repo, stdout, licenses); err != nil { - return err - } + // 6 - update licenses + var licenses []string + if len(opts.License) > 0 { + licenses = append(licenses, opts.License) + + var stdout string + stdout, _, err = git.NewCommand("rev-parse", "HEAD").RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) + if err != nil { + log.Error("CreateRepository(git rev-parse HEAD) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return nil, fmt.Errorf("CreateRepository(git rev-parse HEAD): %w", err) } - return nil - }); err != nil { - if rollbackRepo != nil { - if errDelete := DeleteRepositoryDirectly(ctx, doer, rollbackRepo.ID); errDelete != nil { - log.Error("Rollback deleteRepository: %v", errDelete) - } + if err = repo_model.UpdateRepoLicenses(ctx, repo, stdout, licenses); err != nil { + return nil, err } + } - return nil, err + // 7 - update repository status to be ready + if needsUpdateToReady { + repo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) + } } return repo, nil } -// CreateRepositoryByExample creates a repository for the user/organization. -func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository, overwriteOrAdopt, isFork bool) (err error) { +// createRepositoryInDB creates a repository for the user/organization. +func createRepositoryInDB(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository, isFork bool) (err error) { if err = repo_model.IsUsableRepoName(repo.Name); err != nil { return err } @@ -353,20 +349,6 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re } } - repoPath := repo_model.RepoPath(u.Name, repo.Name) - isExist, err := util.IsExist(repoPath) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return err - } - if !overwriteOrAdopt && isExist { - log.Error("Files already exist in %s and we are not going to adopt or delete.", repoPath) - return repo_model.ErrRepoFilesAlreadyExist{ - Uname: u.Name, - Name: repo.Name, - } - } - if err = db.Insert(ctx, repo); err != nil { return err } @@ -386,7 +368,8 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re } units := make([]repo_model.RepoUnit, 0, len(defaultUnits)) for _, tp := range defaultUnits { - if tp == unit.TypeIssues { + switch tp { + case unit.TypeIssues: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, @@ -396,7 +379,7 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re EnableDependencies: setting.Service.DefaultEnableDependencies, }, }) - } else if tp == unit.TypePullRequests { + case unit.TypePullRequests: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, @@ -406,13 +389,13 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re AllowRebaseUpdate: true, }, }) - } else if tp == unit.TypeProjects { + case unit.TypeProjects: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, Config: &repo_model.ProjectsConfig{ProjectsMode: repo_model.ProjectsModeAll}, }) - } else { + default: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, @@ -474,3 +457,26 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re return nil } + +func cleanupRepository(repoID int64) { + if errDelete := DeleteRepositoryDirectly(db.DefaultContext, repoID); errDelete != nil { + log.Error("cleanupRepository failed: %v", errDelete) + // add system notice + if err := system_model.CreateRepositoryNotice("DeleteRepositoryDirectly failed when cleanup repository: %v", errDelete); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + } +} + +func updateGitRepoAfterCreate(ctx context.Context, repo *repo_model.Repository) error { + if err := checkDaemonExportOK(ctx, repo); err != nil { + return fmt.Errorf("checkDaemonExportOK: %w", err) + } + + if stdout, _, err := git.NewCommand("update-server-info"). + RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}); err != nil { + log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return fmt.Errorf("CreateRepository(git update-server-info): %w", err) + } + return nil +} diff --git a/services/repository/create_test.go b/services/repository/create_test.go new file mode 100644 index 0000000000..fe464c1441 --- /dev/null +++ b/services/repository/create_test.go @@ -0,0 +1,57 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "os" + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" +) + +func TestCreateRepositoryDirectly(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + // a successful creating repository + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + createdRepo, err := CreateRepositoryDirectly(git.DefaultContext, user2, user2, CreateRepoOptions{ + Name: "created-repo", + }, true) + assert.NoError(t, err) + assert.NotNil(t, createdRepo) + + exist, err := util.IsExist(repo_model.RepoPath(user2.Name, createdRepo.Name)) + assert.NoError(t, err) + assert.True(t, exist) + + unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: createdRepo.Name}) + + err = DeleteRepositoryDirectly(db.DefaultContext, createdRepo.ID) + assert.NoError(t, err) + + // a failed creating because some mock data + // create the repository directory so that the creation will fail after database record created. + assert.NoError(t, os.MkdirAll(repo_model.RepoPath(user2.Name, createdRepo.Name), os.ModePerm)) + + createdRepo2, err := CreateRepositoryDirectly(db.DefaultContext, user2, user2, CreateRepoOptions{ + Name: "created-repo", + }, true) + assert.Nil(t, createdRepo2) + assert.Error(t, err) + + // assert the cleanup is successful + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: createdRepo.Name}) + + exist, err = util.IsExist(repo_model.RepoPath(user2.Name, createdRepo.Name)) + assert.NoError(t, err) + assert.False(t, exist) +} diff --git a/services/repository/delete.go b/services/repository/delete.go index 2166b4dd5c..c48d6e1d56 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -14,6 +14,7 @@ import ( git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/organization" + packages_model "code.gitea.io/gitea/models/packages" access_model "code.gitea.io/gitea/models/perm/access" project_model "code.gitea.io/gitea/models/project" repo_model "code.gitea.io/gitea/models/repo" @@ -22,17 +23,33 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" actions_module "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" + actions_service "code.gitea.io/gitea/services/actions" asymkey_service "code.gitea.io/gitea/services/asymkey" + issue_service "code.gitea.io/gitea/services/issue" "xorm.io/builder" ) +func deleteDBRepository(ctx context.Context, repoID int64) error { + if cnt, err := db.GetEngine(ctx).ID(repoID).Delete(&repo_model.Repository{}); err != nil { + return err + } else if cnt != 1 { + return repo_model.ErrRepoNotExist{ + ID: repoID, + OwnerName: "", + Name: "", + } + } + return nil +} + // DeleteRepository deletes a repository for a user or organization. // make sure if you call this func to close open sessions (sqlite will otherwise get a deadlock) -func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID int64, ignoreOrgTeams ...bool) error { +func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams ...bool) error { ctx, committer, err := db.TxContext(ctx) if err != nil { return err @@ -80,14 +97,8 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID } needRewriteKeysFile := deleted > 0 - if cnt, err := sess.ID(repoID).Delete(&repo_model.Repository{}); err != nil { + if err := deleteDBRepository(ctx, repoID); err != nil { return err - } else if cnt != 1 { - return repo_model.ErrRepoNotExist{ - ID: repoID, - OwnerName: "", - Name: "", - } } if org != nil && org.IsOrganization() { @@ -124,6 +135,14 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID return err } + // CleanupEphemeralRunnersByPickedTaskOfRepo deletes ephemeral global/org/user that have started any task of this repo + // The cannot pick a second task hardening for ephemeral runners expect that task objects remain available until runner deletion + // This method will delete affected ephemeral global/org/user runners + // &actions_model.ActionRunner{RepoID: repoID} does only handle ephemeral repository runners + if err := actions_service.CleanupEphemeralRunnersByPickedTaskOfRepo(ctx, repoID); err != nil { + return fmt.Errorf("cleanupEphemeralRunners: %w", err) + } + if err := db.DeleteBeans(ctx, &access_model.Access{RepoID: repo.ID}, &activities_model.Action{RepoID: repo.ID}, @@ -158,6 +177,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID &actions_model.ActionSchedule{RepoID: repoID}, &actions_model.ActionArtifact{RepoID: repoID}, &actions_model.ActionRunnerToken{RepoID: repoID}, + &issues_model.IssuePin{RepoID: repoID}, ); err != nil { return fmt.Errorf("deleteBeans: %w", err) } @@ -174,7 +194,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID // Delete Issues and related objects var attachmentPaths []string - if attachmentPaths, err = issues_model.DeleteIssuesByRepoID(ctx, repoID); err != nil { + if attachmentPaths, err = issue_service.DeleteIssuesByRepoID(ctx, repoID); err != nil { return err } @@ -266,6 +286,11 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID return err } + // unlink packages linked to this repository + if err = packages_model.UnlinkRepositoryFromAllPackages(ctx, repoID); err != nil { + return err + } + if err = committer.Commit(); err != nil { return err } @@ -282,8 +307,13 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID // we delete the file but the database rollback, the repository will be broken. // Remove repository files. - repoPath := repo.RepoPath() - system_model.RemoveAllWithNotice(ctx, "Delete repository files", repoPath) + if err := gitrepo.DeleteRepository(ctx, repo); err != nil { + desc := fmt.Sprintf("Delete repository files [%s]: %v", repo.FullName(), err) + // Note we use the db.DefaultContext here rather than passing in a context as the context may be cancelled + if err = system_model.CreateNotice(db.DefaultContext, system_model.NoticeRepository, desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + } // Remove wiki files if repo.HasWiki() { @@ -345,7 +375,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID // DeleteOwnerRepositoriesDirectly calls DeleteRepositoryDirectly for all repos of the given owner func DeleteOwnerRepositoriesDirectly(ctx context.Context, owner *user_model.User) error { for { - repos, _, err := repo_model.GetUserRepositories(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.GetUserRepositories(ctx, repo_model.SearchRepoOptions{ ListOptions: db.ListOptions{ PageSize: repo_model.RepositoryListDefaultPageSize, Page: 1, @@ -361,7 +391,7 @@ func DeleteOwnerRepositoriesDirectly(ctx context.Context, owner *user_model.User break } for _, repo := range repos { - if err := DeleteRepositoryDirectly(ctx, owner, repo.ID); err != nil { + if err := DeleteRepositoryDirectly(ctx, repo.ID); err != nil { return fmt.Errorf("unable to delete repository %s for %s[%d]. Error: %w", repo.Name, owner.Name, owner.ID, err) } } diff --git a/services/repository/files/cherry_pick.go b/services/repository/files/cherry_pick.go index 10545e9e03..6818bb343d 100644 --- a/services/repository/files/cherry_pick.go +++ b/services/repository/files/cherry_pick.go @@ -5,6 +5,7 @@ package files import ( "context" + "errors" "fmt" "strings" @@ -32,27 +33,25 @@ func (err ErrCommitIDDoesNotMatch) Error() string { return fmt.Sprintf("file CommitID does not match [given: %s, expected: %s]", err.GivenCommitID, err.CurrentCommitID) } -// CherryPick cherrypicks or reverts a commit to the given repository +// CherryPick cherry-picks or reverts a commit to the given repository func CherryPick(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, revert bool, opts *ApplyDiffPatchOptions) (*structs.FileResponse, error) { if err := opts.Validate(ctx, repo, doer); err != nil { return nil, err } message := strings.TrimSpace(opts.Message) - author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) - - t, err := NewTemporaryUploadRepository(ctx, repo) + t, err := NewTemporaryUploadRepository(repo) if err != nil { log.Error("NewTemporaryUploadRepository failed: %v", err) } defer t.Close() - if err := t.Clone(opts.OldBranch, false); err != nil { + if err := t.Clone(ctx, opts.OldBranch, false); err != nil { return nil, err } - if err := t.SetDefaultIndex(); err != nil { + if err := t.SetDefaultIndex(ctx); err != nil { return nil, err } - if err := t.RefreshIndex(); err != nil { + if err := t.RefreshIndex(ctx); err != nil { return nil, err } @@ -102,28 +101,37 @@ func CherryPick(ctx context.Context, repo *repo_model.Repository, doer *user_mod } if conflict { - return nil, fmt.Errorf("failed to merge due to conflicts") + return nil, errors.New("failed to merge due to conflicts") } - treeHash, err := t.WriteTree() + treeHash, err := t.WriteTree(ctx) if err != nil { // likely non-sensical tree due to merge conflicts... return nil, err } // Now commit the tree - var commitHash string + commitOpts := &CommitTreeUserOptions{ + ParentCommitID: "HEAD", + TreeHash: treeHash, + CommitMessage: message, + SignOff: opts.Signoff, + DoerUser: doer, + AuthorIdentity: opts.Author, + AuthorTime: nil, + CommitterIdentity: opts.Committer, + CommitterTime: nil, + } if opts.Dates != nil { - commitHash, err = t.CommitTreeWithDate("HEAD", author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) - } else { - commitHash, err = t.CommitTree("HEAD", author, committer, treeHash, message, opts.Signoff) + commitOpts.AuthorTime, commitOpts.CommitterTime = &opts.Dates.Author, &opts.Dates.Committer } + commitHash, err := t.CommitTree(ctx, commitOpts) if err != nil { return nil, err } // Then push this tree to NewBranch - if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + if err := t.Push(ctx, doer, commitHash, opts.NewBranch); err != nil { return nil, err } diff --git a/services/repository/files/commit.go b/services/repository/files/commit.go index e0dad29273..3cc326d065 100644 --- a/services/repository/files/commit.go +++ b/services/repository/files/commit.go @@ -6,10 +6,10 @@ package files import ( "context" - asymkey_model "code.gitea.io/gitea/models/asymkey" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/structs" + asymkey_service "code.gitea.io/gitea/services/asymkey" ) // CountDivergingCommits determines how many commits a branch is ahead or behind the repository's base branch @@ -24,7 +24,7 @@ func CountDivergingCommits(ctx context.Context, repo *repo_model.Repository, bra // GetPayloadCommitVerification returns the verification information of a commit func GetPayloadCommitVerification(ctx context.Context, commit *git.Commit) *structs.PayloadCommitVerification { verification := &structs.PayloadCommitVerification{} - commitVerification := asymkey_model.ParseCommitWithSignature(ctx, commit) + commitVerification := asymkey_service.ParseCommitWithSignature(ctx, commit) if commit.Signature != nil { verification.Signature = commit.Signature.Signature verification.Payload = commit.Signature.Payload diff --git a/services/repository/files/content.go b/services/repository/files/content.go index 0ab7422ce2..2c1e88bb59 100644 --- a/services/repository/files/content.go +++ b/services/repository/files/content.go @@ -5,17 +5,18 @@ package files import ( "context" - "fmt" + "io" "net/url" "path" "strings" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" ) // ContentType repo content type @@ -23,14 +24,10 @@ type ContentType string // The string representations of different content types const ( - // ContentTypeRegular regular content type (file) - ContentTypeRegular ContentType = "file" - // ContentTypeDir dir content type (dir) - ContentTypeDir ContentType = "dir" - // ContentLink link content type (symlink) - ContentTypeLink ContentType = "symlink" - // ContentTag submodule content type (submodule) - ContentTypeSubmodule ContentType = "submodule" + ContentTypeRegular ContentType = "file" // regular content type (file) + ContentTypeDir ContentType = "dir" // dir content type (dir) + ContentTypeLink ContentType = "symlink" // link content type (symlink) + ContentTypeSubmodule ContentType = "submodule" // submodule content type (submodule) ) // String gets the string of ContentType @@ -38,67 +35,52 @@ func (ct *ContentType) String() string { return string(*ct) } -// GetContentsOrList gets the meta data of a file's contents (*ContentsResponse) if treePath not a tree -// directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag -func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, treePath, ref string) (any, error) { - if repo.IsEmpty { - return make([]any, 0), nil - } - if ref == "" { - ref = repo.DefaultBranch - } - origRef := ref - - // Check that the path given in opts.treePath is valid (not a git path) - cleanTreePath := CleanUploadFileName(treePath) - if cleanTreePath == "" && treePath != "" { - return nil, ErrFilenameInvalid{ - Path: treePath, - } - } - treePath = cleanTreePath - - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return nil, err - } - defer closer.Close() +type GetContentsOrListOptions struct { + TreePath string + IncludeSingleFileContent bool // include the file's content when the tree path is a file + IncludeLfsMetadata bool + IncludeCommitMetadata bool + IncludeCommitMessage bool +} - // Get the commit object for the ref - commit, err := gitRepo.GetCommit(ref) - if err != nil { - return nil, err +// GetContentsOrList gets the metadata of a file's contents (*ContentsResponse) if treePath not a tree +// directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag +func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, opts GetContentsOrListOptions) (ret api.ContentsExtResponse, _ error) { + entry, err := prepareGetContentsEntry(refCommit, &opts.TreePath) + if repo.IsEmpty && opts.TreePath == "" { + return api.ContentsExtResponse{DirContents: make([]*api.ContentsResponse, 0)}, nil } - - entry, err := commit.GetTreeEntryByPath(treePath) if err != nil { - return nil, err + return ret, err } + // get file contents if entry.Type() != "tree" { - return GetContents(ctx, repo, treePath, origRef, false) + ret.FileContents, err = getFileContentsByEntryInternal(ctx, repo, gitRepo, refCommit, entry, opts) + return ret, err } - // We are in a directory, so we return a list of FileContentResponse objects - var fileList []*api.ContentsResponse - - gitTree, err := commit.SubTree(treePath) + // list directory contents + gitTree, err := refCommit.Commit.SubTree(opts.TreePath) if err != nil { - return nil, err + return ret, err } entries, err := gitTree.ListEntries() if err != nil { - return nil, err + return ret, err } + ret.DirContents = make([]*api.ContentsResponse, 0, len(entries)) for _, e := range entries { - subTreePath := path.Join(treePath, e.Name()) - fileContentResponse, err := GetContents(ctx, repo, subTreePath, origRef, true) + subOpts := opts + subOpts.TreePath = path.Join(opts.TreePath, e.Name()) + subOpts.IncludeSingleFileContent = false // never include file content when listing a directory + fileContentResponse, err := GetFileContents(ctx, repo, gitRepo, refCommit, subOpts) if err != nil { - return nil, err + return ret, err } - fileList = append(fileList, fileContentResponse) + ret.DirContents = append(ret.DirContents, fileContentResponse) } - return fileList, nil + return ret, nil } // GetObjectTypeFromTreeEntry check what content is behind it @@ -117,86 +99,96 @@ func GetObjectTypeFromTreeEntry(entry *git.TreeEntry) ContentType { } } -// GetContents gets the meta data on a file's contents. Ref can be a branch, commit or tag -func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref string, forList bool) (*api.ContentsResponse, error) { - if ref == "" { - ref = repo.DefaultBranch - } - origRef := ref - +func prepareGetContentsEntry(refCommit *utils.RefCommit, treePath *string) (*git.TreeEntry, error) { // Check that the path given in opts.treePath is valid (not a git path) - cleanTreePath := CleanUploadFileName(treePath) - if cleanTreePath == "" && treePath != "" { - return nil, ErrFilenameInvalid{ - Path: treePath, - } + cleanTreePath := CleanGitTreePath(*treePath) + if cleanTreePath == "" && *treePath != "" { + return nil, ErrFilenameInvalid{Path: *treePath} } - treePath = cleanTreePath + *treePath = cleanTreePath - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return nil, err + // Only allow safe ref types + refType := refCommit.RefName.RefType() + if refType != git.RefTypeBranch && refType != git.RefTypeTag && refType != git.RefTypeCommit { + return nil, util.NewNotExistErrorf("no commit found for the ref [ref: %s]", refCommit.RefName) } - defer closer.Close() - // Get the commit object for the ref - commit, err := gitRepo.GetCommit(ref) - if err != nil { - return nil, err - } - commitID := commit.ID.String() - if len(ref) >= 4 && strings.HasPrefix(commitID, ref) { - ref = commit.ID.String() - } + return refCommit.Commit.GetTreeEntryByPath(*treePath) +} - entry, err := commit.GetTreeEntryByPath(treePath) +// GetFileContents gets the metadata on a file's contents. Ref can be a branch, commit or tag +func GetFileContents(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, opts GetContentsOrListOptions) (*api.ContentsResponse, error) { + entry, err := prepareGetContentsEntry(refCommit, &opts.TreePath) if err != nil { return nil, err } + return getFileContentsByEntryInternal(ctx, repo, gitRepo, refCommit, entry, opts) +} - refType := gitRepo.GetRefType(ref) - if refType == "invalid" { - return nil, fmt.Errorf("no commit found for the ref [ref: %s]", ref) - } - - selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(treePath) + "?ref=" + url.QueryEscape(origRef)) +func getFileContentsByEntryInternal(_ context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, entry *git.TreeEntry, opts GetContentsOrListOptions) (*api.ContentsResponse, error) { + refType := refCommit.RefName.RefType() + commit := refCommit.Commit + selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(opts.TreePath) + "?ref=" + url.QueryEscape(refCommit.InputRef)) if err != nil { return nil, err } selfURLString := selfURL.String() - err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(ref, refType != git.ObjectCommit), repo.FullName(), commitID) - if err != nil { - return nil, err - } - - lastCommit, err := commit.GetCommitByPath(treePath) - if err != nil { - return nil, err - } - // All content types have these fields in populated contentsResponse := &api.ContentsResponse{ - Name: entry.Name(), - Path: treePath, - SHA: entry.ID.String(), - LastCommitSHA: lastCommit.ID.String(), - Size: entry.Size(), - URL: &selfURLString, + Name: entry.Name(), + Path: opts.TreePath, + SHA: entry.ID.String(), + Size: entry.Size(), + URL: &selfURLString, Links: &api.FileLinksResponse{ Self: &selfURLString, }, } - // Now populate the rest of the ContentsResponse based on entry type + if opts.IncludeCommitMetadata || opts.IncludeCommitMessage { + err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(refCommit.InputRef, refType != git.RefTypeCommit), repo.FullName(), refCommit.CommitID) + if err != nil { + return nil, err + } + + lastCommit, err := refCommit.Commit.GetCommitByPath(opts.TreePath) + if err != nil { + return nil, err + } + + if opts.IncludeCommitMetadata { + contentsResponse.LastCommitSHA = util.ToPointer(lastCommit.ID.String()) + // GitHub doesn't have these fields in the response, but we could follow other similar APIs to name them + // https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits + if lastCommit.Committer != nil { + contentsResponse.LastCommitterDate = util.ToPointer(lastCommit.Committer.When) + } + if lastCommit.Author != nil { + contentsResponse.LastAuthorDate = util.ToPointer(lastCommit.Author.When) + } + } + if opts.IncludeCommitMessage { + contentsResponse.LastCommitMessage = util.ToPointer(lastCommit.Message()) + } + } + + // Now populate the rest of the ContentsResponse based on the entry type if entry.IsRegular() || entry.IsExecutable() { contentsResponse.Type = string(ContentTypeRegular) - if blobResponse, err := GetBlobBySHA(ctx, repo, gitRepo, entry.ID.String()); err != nil { - return nil, err - } else if !forList { - // We don't show the content if we are getting a list of FileContentResponses - contentsResponse.Encoding = &blobResponse.Encoding - contentsResponse.Content = &blobResponse.Content + // if it is listing the repo root dir, don't waste system resources on reading content + if opts.IncludeSingleFileContent { + blobResponse, err := GetBlobBySHA(repo, gitRepo, entry.ID.String()) + if err != nil { + return nil, err + } + contentsResponse.Encoding, contentsResponse.Content = blobResponse.Encoding, blobResponse.Content + contentsResponse.LfsOid, contentsResponse.LfsSize = blobResponse.LfsOid, blobResponse.LfsSize + } else if opts.IncludeLfsMetadata { + contentsResponse.LfsOid, contentsResponse.LfsSize, err = parsePossibleLfsPointerBlob(gitRepo, entry.ID.String()) + if err != nil { + return nil, err + } } } else if entry.IsDir() { contentsResponse.Type = string(ContentTypeDir) @@ -210,7 +202,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref contentsResponse.Target = &targetFromContent } else if entry.IsSubModule() { contentsResponse.Type = string(ContentTypeSubmodule) - submodule, err := commit.GetSubModule(treePath) + submodule, err := commit.GetSubModule(opts.TreePath) if err != nil { return nil, err } @@ -220,7 +212,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref } // Handle links if entry.IsRegular() || entry.IsLink() || entry.IsExecutable() { - downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(opts.TreePath)) if err != nil { return nil, err } @@ -228,7 +220,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref contentsResponse.DownloadURL = &downloadURLString } if !entry.IsSubModule() { - htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(opts.TreePath)) if err != nil { return nil, err } @@ -248,49 +240,59 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref return contentsResponse, nil } -// GetBlobBySHA get the GitBlobResponse of a repository using a sha hash. -func GetBlobBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) { +func GetBlobBySHA(repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) { gitBlob, err := gitRepo.GetBlob(sha) if err != nil { return nil, err } - content := "" - if gitBlob.Size() <= setting.API.DefaultMaxBlobSize { - content, err = gitBlob.GetBlobContentBase64() - if err != nil { - return nil, err - } + ret := &api.GitBlobResponse{ + SHA: gitBlob.ID.String(), + URL: repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()), + Size: gitBlob.Size(), } - return &api.GitBlobResponse{ - SHA: gitBlob.ID.String(), - URL: repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()), - Size: gitBlob.Size(), - Encoding: "base64", - Content: content, - }, nil -} -// TryGetContentLanguage tries to get the (linguist) language of the file content -func TryGetContentLanguage(gitRepo *git.Repository, commitID, treePath string) (string, error) { - indexFilename, worktree, deleteTemporaryFile, err := gitRepo.ReadTreeToTemporaryIndex(commitID) - if err != nil { - return "", err + blobSize := gitBlob.Size() + if blobSize > setting.API.DefaultMaxBlobSize { + return ret, nil } - defer deleteTemporaryFile() + var originContent *strings.Builder + if 0 < blobSize && blobSize < lfs.MetaFileMaxSize { + originContent = &strings.Builder{} + } - filename2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{ - CachedOnly: true, - Attributes: []string{git.AttributeLinguistLanguage, git.AttributeGitlabLanguage}, - Filenames: []string{treePath}, - IndexFile: indexFilename, - WorkTree: worktree, - }) + content, err := gitBlob.GetBlobContentBase64(originContent) if err != nil { - return "", err + return nil, err + } + + ret.Encoding, ret.Content = util.ToPointer("base64"), &content + if originContent != nil { + ret.LfsOid, ret.LfsSize = parsePossibleLfsPointerBuffer(strings.NewReader(originContent.String())) } + return ret, nil +} - language := git.TryReadLanguageAttribute(filename2attribute2info[treePath]) +func parsePossibleLfsPointerBuffer(r io.Reader) (*string, *int64) { + p, _ := lfs.ReadPointer(r) + if p.IsValid() { + return &p.Oid, &p.Size + } + return nil, nil +} - return language.Value(), nil +func parsePossibleLfsPointerBlob(gitRepo *git.Repository, sha string) (*string, *int64, error) { + gitBlob, err := gitRepo.GetBlob(sha) + if err != nil { + return nil, nil, err + } + if gitBlob.Size() > lfs.MetaFileMaxSize { + return nil, nil, nil // not a LFS pointer + } + buf, err := gitBlob.GetBlobContent(lfs.MetaFileMaxSize) + if err != nil { + return nil, nil, err + } + oid, size := parsePossibleLfsPointerBuffer(strings.NewReader(buf)) + return oid, size, nil } diff --git a/services/repository/files/content_test.go b/services/repository/files/content_test.go index 7cb46c0bb6..d72f918074 100644 --- a/services/repository/files/content_test.go +++ b/services/repository/files/content_test.go @@ -7,8 +7,8 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/gitrepo" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/contexttest" _ "code.gitea.io/gitea/models/actions" @@ -20,36 +20,6 @@ func TestMain(m *testing.M) { unittest.MainTest(m) } -func getExpectedReadmeContentsResponse() *api.ContentsResponse { - treePath := "README.md" - sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" - encoding := "base64" - content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" - selfURL := "https://try.gitea.io/api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" - htmlURL := "https://try.gitea.io/user2/repo1/src/branch/master/" + treePath - gitURL := "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/" + sha - downloadURL := "https://try.gitea.io/user2/repo1/raw/branch/master/" + treePath - return &api.ContentsResponse{ - Name: treePath, - Path: treePath, - SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", - LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - Type: "file", - Size: 30, - Encoding: &encoding, - Content: &content, - URL: &selfURL, - HTMLURL: &htmlURL, - GitURL: &gitURL, - DownloadURL: &downloadURL, - Links: &api.FileLinksResponse{ - Self: &selfURL, - GitURL: &gitURL, - HTMLURL: &htmlURL, - }, - } -} - func TestGetContents(t *testing.T) { unittest.PrepareTestEnv(t) ctx, _ := contexttest.MockContext(t, "user2/repo1") @@ -58,195 +28,22 @@ func TestGetContents(t *testing.T) { contexttest.LoadRepoCommit(t, ctx) contexttest.LoadUser(t, ctx, 2) contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - treePath := "README.md" - ref := ctx.Repo.Repository.DefaultBranch - - expectedContentsResponse := getExpectedReadmeContentsResponse() - - t.Run("Get README.md contents with GetContents(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContents(ctx, ctx.Repo.Repository, treePath, ref, false) - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContents(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContents(ctx, ctx.Repo.Repository, treePath, "", false) - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) -} - -func TestGetContentsOrListForDir(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - treePath := "" // root dir - ref := ctx.Repo.Repository.DefaultBranch - - readmeContentsResponse := getExpectedReadmeContentsResponse() - // because will be in a list, doesn't have encoding and content - readmeContentsResponse.Encoding = nil - readmeContentsResponse.Content = nil - - expectedContentsListResponse := []*api.ContentsResponse{ - readmeContentsResponse, - } - - t.Run("Get root dir contents with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, ref) - assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get root dir contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, "") - assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) - assert.NoError(t, err) - }) -} - -func TestGetContentsOrListForFile(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - treePath := "README.md" - ref := ctx.Repo.Repository.DefaultBranch - - expectedContentsResponse := getExpectedReadmeContentsResponse() - - t.Run("Get README.md contents with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, ref) - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, "") - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) -} - -func TestGetContentsErrors(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - treePath := "README.md" - ref := repo.DefaultBranch - - t.Run("bad treePath", func(t *testing.T) { - badTreePath := "bad/tree.md" - fileContentResponse, err := GetContents(ctx, repo, badTreePath, ref, false) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: , rel_path: bad]") - assert.Nil(t, fileContentResponse) - }) - - t.Run("bad ref", func(t *testing.T) { - badRef := "bad_ref" - fileContentResponse, err := GetContents(ctx, repo, treePath, badRef, false) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") - assert.Nil(t, fileContentResponse) - }) -} - -func TestGetContentsOrListErrors(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - repo := ctx.Repo.Repository - treePath := "README.md" - ref := repo.DefaultBranch - - t.Run("bad treePath", func(t *testing.T) { - badTreePath := "bad/tree.md" - fileContentResponse, err := GetContentsOrList(ctx, repo, badTreePath, ref) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: , rel_path: bad]") - assert.Nil(t, fileContentResponse) - }) - - t.Run("bad ref", func(t *testing.T) { - badRef := "bad_ref" - fileContentResponse, err := GetContentsOrList(ctx, repo, treePath, badRef) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") - assert.Nil(t, fileContentResponse) - }) -} - -func TestGetContentsOrListOfEmptyRepos(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user30/empty") - ctx.SetPathParam("id", "52") - contexttest.LoadRepo(t, ctx, 52) - contexttest.LoadUser(t, ctx, 30) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - - t.Run("empty repo", func(t *testing.T) { - contents, err := GetContentsOrList(ctx, repo, "", "") + // GetContentsOrList's behavior is fully tested in integration tests, so we don't need to test it here. + + t.Run("GetBlobBySHA", func(t *testing.T) { + sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" + ctx.SetPathParam("id", "1") + ctx.SetPathParam("sha", sha) + gbr, err := GetBlobBySHA(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("sha")) + expectedGBR := &api.GitBlobResponse{ + Content: util.ToPointer("dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK"), + Encoding: util.ToPointer("base64"), + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + Size: 180, + } assert.NoError(t, err) - assert.Empty(t, contents) + assert.Equal(t, expectedGBR, gbr) }) } - -func TestGetBlobBySHA(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" - ctx.SetPathParam("id", "1") - ctx.SetPathParam("sha", sha) - - gitRepo, err := gitrepo.OpenRepository(ctx, ctx.Repo.Repository) - if err != nil { - t.Fail() - } - - gbr, err := GetBlobBySHA(ctx, ctx.Repo.Repository, gitRepo, ctx.PathParam("sha")) - expectedGBR := &api.GitBlobResponse{ - Content: "dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK", - Encoding: "base64", - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", - SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - Size: 180, - } - assert.NoError(t, err) - assert.Equal(t, expectedGBR, gbr) -} diff --git a/services/repository/files/diff.go b/services/repository/files/diff.go index bf8b938e21..50d01f9d7c 100644 --- a/services/repository/files/diff.go +++ b/services/repository/files/diff.go @@ -16,27 +16,27 @@ func GetDiffPreview(ctx context.Context, repo *repo_model.Repository, branch, tr if branch == "" { branch = repo.DefaultBranch } - t, err := NewTemporaryUploadRepository(ctx, repo) + t, err := NewTemporaryUploadRepository(repo) if err != nil { return nil, err } defer t.Close() - if err := t.Clone(branch, true); err != nil { + if err := t.Clone(ctx, branch, true); err != nil { return nil, err } - if err := t.SetDefaultIndex(); err != nil { + if err := t.SetDefaultIndex(ctx); err != nil { return nil, err } // Add the object to the database - objectHash, err := t.HashObject(strings.NewReader(content)) + objectHash, err := t.HashObjectAndWrite(ctx, strings.NewReader(content)) if err != nil { return nil, err } // Add the object to the index - if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil { + if err := t.AddObjectToIndex(ctx, "100644", objectHash, treePath); err != nil { return nil, err } - return t.DiffIndex() + return t.DiffIndex(ctx) } diff --git a/services/repository/files/diff_test.go b/services/repository/files/diff_test.go index b7bdcd8ecf..ae702e4189 100644 --- a/services/repository/files/diff_test.go +++ b/services/repository/files/diff_test.go @@ -30,14 +30,11 @@ func TestGetDiffPreview(t *testing.T) { content := "# repo1\n\nDescription for repo1\nthis is a new line" expectedDiff := &gitdiff.Diff{ - TotalAddition: 2, - TotalDeletion: 1, Files: []*gitdiff.DiffFile{ { Name: "README.md", OldName: "README.md", NameHash: "8ec9a00bfd09b3190ac6b22251dbb1aa95a0579d", - Index: 1, Addition: 2, Deletion: 1, Type: 2, @@ -50,7 +47,6 @@ func TestGetDiffPreview(t *testing.T) { Sections: []*gitdiff.DiffSection{ { FileName: "README.md", - Name: "", Lines: []*gitdiff.DiffLine{ { LeftIdx: 0, @@ -114,7 +110,6 @@ func TestGetDiffPreview(t *testing.T) { }, IsIncomplete: false, } - expectedDiff.NumFiles = len(expectedDiff.Files) t.Run("with given branch", func(t *testing.T) { diff, err := GetDiffPreview(ctx, ctx.Repo.Repository, branch, treePath, content) @@ -123,7 +118,7 @@ func TestGetDiffPreview(t *testing.T) { assert.NoError(t, err) bs, err := json.Marshal(diff) assert.NoError(t, err) - assert.EqualValues(t, string(expectedBs), string(bs)) + assert.Equal(t, string(expectedBs), string(bs)) }) t.Run("empty branch, same results", func(t *testing.T) { @@ -133,7 +128,7 @@ func TestGetDiffPreview(t *testing.T) { assert.NoError(t, err) bs, err := json.Marshal(diff) assert.NoError(t, err) - assert.EqualValues(t, expectedBs, bs) + assert.Equal(t, expectedBs, bs) }) } diff --git a/services/repository/files/file.go b/services/repository/files/file.go index d7ca8e79e5..13d171d139 100644 --- a/services/repository/files/file.go +++ b/services/repository/files/file.go @@ -5,26 +5,48 @@ package files import ( "context" + "errors" "fmt" "net/url" "strings" "time" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" ) -func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch string, treeNames []string) (*api.FilesResponse, error) { - files := []*api.ContentsResponse{} - for _, file := range treeNames { - fileContents, _ := GetContents(ctx, repo, file, branch, false) // ok if fails, then will be nil +func GetContentsListFromTreePaths(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, treePaths []string) (files []*api.ContentsResponse) { + var size int64 + for _, treePath := range treePaths { + // ok if fails, then will be nil + fileContents, _ := GetFileContents(ctx, repo, gitRepo, refCommit, GetContentsOrListOptions{ + TreePath: treePath, + IncludeSingleFileContent: true, + IncludeCommitMetadata: true, + }) + if fileContents != nil && fileContents.Content != nil && *fileContents.Content != "" { + // if content isn't empty (e.g., due to the single blob being too large), add file size to response size + size += int64(len(*fileContents.Content)) + } + if size > setting.API.DefaultMaxResponseSize { + break // stop if max response size would be exceeded + } files = append(files, fileContents) + if len(files) == setting.API.DefaultPagingNum { + break // stop if paging num reached + } } - fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil - verification := GetPayloadCommitVerification(ctx, commit) + return files +} + +func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, treeNames []string) (*api.FilesResponse, error) { + files := GetContentsListFromTreePaths(ctx, repo, gitRepo, refCommit, treeNames) + fileCommitResponse, _ := GetFileCommitResponse(repo, refCommit.Commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, refCommit.Commit) filesResponse := &api.FilesResponse{ Files: files, Commit: fileCommitResponse, @@ -33,19 +55,6 @@ func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository return filesResponse, nil } -// GetFileResponseFromCommit Constructs a FileResponse from a Commit object -func GetFileResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch, treeName string) (*api.FileResponse, error) { - fileContents, _ := GetContents(ctx, repo, treeName, branch, false) // ok if fails, then will be nil - fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil - verification := GetPayloadCommitVerification(ctx, commit) - fileResponse := &api.FileResponse{ - Content: fileContents, - Commit: fileCommitResponse, - Verification: verification, - } - return fileResponse, nil -} - // constructs a FileResponse with the file at the index from FilesResponse func GetFileResponseFromFilesResponse(filesResponse *api.FilesResponse, index int) *api.FileResponse { content := &api.ContentsResponse{} @@ -63,10 +72,10 @@ func GetFileResponseFromFilesResponse(filesResponse *api.FilesResponse, index in // GetFileCommitResponse Constructs a FileCommitResponse from a Commit object func GetFileCommitResponse(repo *repo_model.Repository, commit *git.Commit) (*api.FileCommitResponse, error) { if repo == nil { - return nil, fmt.Errorf("repo cannot be nil") + return nil, errors.New("repo cannot be nil") } if commit == nil { - return nil, fmt.Errorf("commit cannot be nil") + return nil, errors.New("commit cannot be nil") } commitURL, _ := url.Parse(repo.APIURL() + "/git/commits/" + url.PathEscape(commit.ID.String())) commitTreeURL, _ := url.Parse(repo.APIURL() + "/git/trees/" + url.PathEscape(commit.Tree.ID.String())) @@ -111,51 +120,6 @@ func GetFileCommitResponse(repo *repo_model.Repository, commit *git.Commit) (*ap return fileCommit, nil } -// GetAuthorAndCommitterUsers Gets the author and committer user objects from the IdentityOptions -func GetAuthorAndCommitterUsers(author, committer *IdentityOptions, doer *user_model.User) (authorUser, committerUser *user_model.User) { - // Committer and author are optional. If they are not the doer (not same email address) - // then we use bogus User objects for them to store their FullName and Email. - // If only one of the two are provided, we set both of them to it. - // If neither are provided, both are the doer. - if committer != nil && committer.Email != "" { - if doer != nil && strings.EqualFold(doer.Email, committer.Email) { - committerUser = doer // the committer is the doer, so will use their user object - if committer.Name != "" { - committerUser.FullName = committer.Name - } - } else { - committerUser = &user_model.User{ - FullName: committer.Name, - Email: committer.Email, - } - } - } - if author != nil && author.Email != "" { - if doer != nil && strings.EqualFold(doer.Email, author.Email) { - authorUser = doer // the author is the doer, so will use their user object - if authorUser.Name != "" { - authorUser.FullName = author.Name - } - } else { - authorUser = &user_model.User{ - FullName: author.Name, - Email: author.Email, - } - } - } - if authorUser == nil { - if committerUser != nil { - authorUser = committerUser // No valid author was given so use the committer - } else if doer != nil { - authorUser = doer // No valid author was given and no valid committer so use the doer - } - } - if committerUser == nil { - committerUser = authorUser // No valid committer so use the author as the committer (was set to a valid user above) - } - return authorUser, committerUser -} - // ErrFilenameInvalid represents a "FilenameInvalid" kind of error. type ErrFilenameInvalid struct { Path string @@ -175,15 +139,17 @@ func (err ErrFilenameInvalid) Unwrap() error { return util.ErrInvalidArgument } -// CleanUploadFileName Trims a filename and returns empty string if it is a .git directory -func CleanUploadFileName(name string) string { - // Rebase the filename +// CleanGitTreePath cleans a tree path for git, it returns an empty string the path is invalid (e.g.: contains ".git" part) +func CleanGitTreePath(name string) string { name = util.PathJoinRel(name) // Git disallows any filenames to have a .git directory in them. - for _, part := range strings.Split(name, "/") { + for part := range strings.SplitSeq(name, "/") { if strings.ToLower(part) == ".git" { return "" } } + if name == "." { + name = "" + } return name } diff --git a/services/repository/files/file_test.go b/services/repository/files/file_test.go index 52c0574883..cdb6a266ff 100644 --- a/services/repository/files/file_test.go +++ b/services/repository/files/file_test.go @@ -6,115 +6,22 @@ package files import ( "testing" - "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/gitrepo" - "code.gitea.io/gitea/modules/setting" - api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/services/contexttest" - "github.com/stretchr/testify/assert" ) func TestCleanUploadFileName(t *testing.T) { - t.Run("Clean regular file", func(t *testing.T) { - name := "this/is/test" - cleanName := CleanUploadFileName(name) - expectedCleanName := name - assert.EqualValues(t, expectedCleanName, cleanName) - }) - - t.Run("Clean a .git path", func(t *testing.T) { - name := "this/is/test/.git" - cleanName := CleanUploadFileName(name) - expectedCleanName := "" - assert.EqualValues(t, expectedCleanName, cleanName) - }) -} - -func getExpectedFileResponse() *api.FileResponse { - treePath := "README.md" - sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" - encoding := "base64" - content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" - selfURL := setting.AppURL + "api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" - htmlURL := setting.AppURL + "user2/repo1/src/branch/master/" + treePath - gitURL := setting.AppURL + "api/v1/repos/user2/repo1/git/blobs/" + sha - downloadURL := setting.AppURL + "user2/repo1/raw/branch/master/" + treePath - return &api.FileResponse{ - Content: &api.ContentsResponse{ - Name: treePath, - Path: treePath, - SHA: sha, - LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - Type: "file", - Size: 30, - Encoding: &encoding, - Content: &content, - URL: &selfURL, - HTMLURL: &htmlURL, - GitURL: &gitURL, - DownloadURL: &downloadURL, - Links: &api.FileLinksResponse{ - Self: &selfURL, - GitURL: &gitURL, - HTMLURL: &htmlURL, - }, - }, - Commit: &api.FileCommitResponse{ - CommitMeta: api.CommitMeta{ - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/65f1bf27bc3bf70f64657658635e66094edbcb4d", - SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - }, - HTMLURL: "https://try.gitea.io/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d", - Author: &api.CommitUser{ - Identity: api.Identity{ - Name: "user1", - Email: "address1@example.com", - }, - Date: "2017-03-19T20:47:59Z", - }, - Committer: &api.CommitUser{ - Identity: api.Identity{ - Name: "Ethan Koenig", - Email: "ethantkoenig@gmail.com", - }, - Date: "2017-03-19T20:47:59Z", - }, - Parents: []*api.CommitMeta{}, - Message: "Initial commit\n", - Tree: &api.CommitMeta{ - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/2a2f1d4670728a2e10049e345bd7a276468beab6", - SHA: "2a2f1d4670728a2e10049e345bd7a276468beab6", - }, - }, - Verification: &api.PayloadCommitVerification{ - Verified: false, - Reason: "gpg.error.not_signed_commit", - Signature: "", - Payload: "", - }, + cases := []struct { + input, expected string + }{ + {"", ""}, + {".", ""}, + {"a/./b", "a/b"}, + {"a.git", "a.git"}, + {".git/b", ""}, + {"a/.git", ""}, + {"/a/../../b", "b"}, + } + for _, c := range cases { + assert.Equal(t, c.expected, CleanGitTreePath(c.input), "input: %q", c.input) } -} - -func TestGetFileResponseFromCommit(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - branch := repo.DefaultBranch - treePath := "README.md" - gitRepo, _ := gitrepo.OpenRepository(ctx, repo) - defer gitRepo.Close() - commit, _ := gitRepo.GetBranchCommit(branch) - expectedFileResponse := getExpectedFileResponse() - - fileResponse, err := GetFileResponseFromCommit(ctx, repo, commit, branch, treePath) - assert.NoError(t, err) - assert.EqualValues(t, expectedFileResponse, fileResponse) } diff --git a/services/repository/files/patch.go b/services/repository/files/patch.go index 38c17b4073..11a8744b7f 100644 --- a/services/repository/files/patch.go +++ b/services/repository/files/patch.go @@ -12,7 +12,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -45,7 +44,6 @@ type ApplyDiffPatchOptions struct { NewBranch string Message string Content string - SHA string Author *IdentityOptions Committer *IdentityOptions Dates *CommitDateOptions @@ -62,29 +60,26 @@ func (opts *ApplyDiffPatchOptions) Validate(ctx context.Context, repo *repo_mode opts.NewBranch = opts.OldBranch } - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return err - } - defer closer.Close() - // oldBranch must exist for this operation - if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil { + if exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.OldBranch); err != nil { return err + } else if !exist { + return git_model.ErrBranchNotExist{ + BranchName: opts.OldBranch, + } } // A NewBranch can be specified for the patch to be applied to. // Check to make sure the branch does not already exist, otherwise we can't proceed. // If we aren't branching to a new branch, make sure user can commit to the given branch if opts.NewBranch != opts.OldBranch { - existingBranch, err := gitRepo.GetBranch(opts.NewBranch) - if existingBranch != nil { + exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.NewBranch) + if err != nil { + return err + } else if exist { return git_model.ErrBranchAlreadyExists{ BranchName: opts.NewBranch, } } - if err != nil && !git.IsErrBranchNotExist(err) { - return err - } } else { protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, opts.OldBranch) if err != nil { @@ -126,17 +121,15 @@ func ApplyDiffPatch(ctx context.Context, repo *repo_model.Repository, doer *user message := strings.TrimSpace(opts.Message) - author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) - - t, err := NewTemporaryUploadRepository(ctx, repo) + t, err := NewTemporaryUploadRepository(repo) if err != nil { log.Error("NewTemporaryUploadRepository failed: %v", err) } defer t.Close() - if err := t.Clone(opts.OldBranch, true); err != nil { + if err := t.Clone(ctx, opts.OldBranch, true); err != nil { return nil, err } - if err := t.SetDefaultIndex(); err != nil { + if err := t.SetDefaultIndex(ctx); err != nil { return nil, err } @@ -166,12 +159,12 @@ func ApplyDiffPatch(ctx context.Context, repo *repo_model.Repository, doer *user stdout := &strings.Builder{} stderr := &strings.Builder{} - cmdApply := git.NewCommand(ctx, "apply", "--index", "--recount", "--cached", "--ignore-whitespace", "--whitespace=fix", "--binary") + cmdApply := git.NewCommand("apply", "--index", "--recount", "--cached", "--ignore-whitespace", "--whitespace=fix", "--binary") if git.DefaultFeatures().CheckVersionAtLeast("2.32") { cmdApply.AddArguments("-3") } - if err := cmdApply.Run(&git.RunOpts{ + if err := cmdApply.Run(ctx, &git.RunOpts{ Dir: t.basePath, Stdout: stdout, Stderr: stderr, @@ -181,24 +174,33 @@ func ApplyDiffPatch(ctx context.Context, repo *repo_model.Repository, doer *user } // Now write the tree - treeHash, err := t.WriteTree() + treeHash, err := t.WriteTree(ctx) if err != nil { return nil, err } // Now commit the tree - var commitHash string + commitOpts := &CommitTreeUserOptions{ + ParentCommitID: "HEAD", + TreeHash: treeHash, + CommitMessage: message, + SignOff: opts.Signoff, + DoerUser: doer, + AuthorIdentity: opts.Author, + AuthorTime: nil, + CommitterIdentity: opts.Committer, + CommitterTime: nil, + } if opts.Dates != nil { - commitHash, err = t.CommitTreeWithDate("HEAD", author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) - } else { - commitHash, err = t.CommitTree("HEAD", author, committer, treeHash, message, opts.Signoff) + commitOpts.AuthorTime, commitOpts.CommitterTime = &opts.Dates.Author, &opts.Dates.Committer } + commitHash, err := t.CommitTree(ctx, commitOpts) if err != nil { return nil, err } // Then push this tree to NewBranch - if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + if err := t.Push(ctx, doer, commitHash, opts.NewBranch); err != nil { return nil, err } diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go index 138af991f9..c2f61c8223 100644 --- a/services/repository/files/temp_repo.go +++ b/services/repository/files/temp_repo.go @@ -6,6 +6,7 @@ package files import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -19,44 +20,45 @@ import ( "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" "code.gitea.io/gitea/services/gitdiff" ) // TemporaryUploadRepository is a type to wrap our upload repositories as a shallow clone type TemporaryUploadRepository struct { - ctx context.Context repo *repo_model.Repository gitRepo *git.Repository basePath string + cleanup func() } // NewTemporaryUploadRepository creates a new temporary upload repository -func NewTemporaryUploadRepository(ctx context.Context, repo *repo_model.Repository) (*TemporaryUploadRepository, error) { - basePath, err := repo_module.CreateTemporaryPath("upload") +func NewTemporaryUploadRepository(repo *repo_model.Repository) (*TemporaryUploadRepository, error) { + basePath, cleanup, err := repo_module.CreateTemporaryPath("upload") if err != nil { return nil, err } - t := &TemporaryUploadRepository{ctx: ctx, repo: repo, basePath: basePath} + t := &TemporaryUploadRepository{repo: repo, basePath: basePath, cleanup: cleanup} return t, nil } // Close the repository cleaning up all files func (t *TemporaryUploadRepository) Close() { defer t.gitRepo.Close() - if err := repo_module.RemoveTemporaryPath(t.basePath); err != nil { - log.Error("Failed to remove temporary path %s: %v", t.basePath, err) + if t.cleanup != nil { + t.cleanup() } } // Clone the base repository to our path and set branch as the HEAD -func (t *TemporaryUploadRepository) Clone(branch string, bare bool) error { - cmd := git.NewCommand(t.ctx, "clone", "-s", "-b").AddDynamicArguments(branch, t.repo.RepoPath(), t.basePath) +func (t *TemporaryUploadRepository) Clone(ctx context.Context, branch string, bare bool) error { + cmd := git.NewCommand("clone", "-s", "-b").AddDynamicArguments(branch, t.repo.RepoPath(), t.basePath) if bare { cmd.AddArguments("--bare") } - if _, _, err := cmd.RunStdString(nil); err != nil { + if _, _, err := cmd.RunStdString(ctx, nil); err != nil { stderr := err.Error() if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched { return git.ErrBranchNotExist{ @@ -72,7 +74,7 @@ func (t *TemporaryUploadRepository) Clone(branch string, bare bool) error { } return fmt.Errorf("Clone: %w %s", err, stderr) } - gitRepo, err := git.OpenRepository(t.ctx, t.basePath) + gitRepo, err := git.OpenRepository(ctx, t.basePath) if err != nil { return err } @@ -81,11 +83,11 @@ func (t *TemporaryUploadRepository) Clone(branch string, bare bool) error { } // Init the repository -func (t *TemporaryUploadRepository) Init(objectFormatName string) error { - if err := git.InitRepository(t.ctx, t.basePath, false, objectFormatName); err != nil { +func (t *TemporaryUploadRepository) Init(ctx context.Context, objectFormatName string) error { + if err := git.InitRepository(ctx, t.basePath, false, objectFormatName); err != nil { return err } - gitRepo, err := git.OpenRepository(t.ctx, t.basePath) + gitRepo, err := git.OpenRepository(ctx, t.basePath) if err != nil { return err } @@ -94,28 +96,28 @@ func (t *TemporaryUploadRepository) Init(objectFormatName string) error { } // SetDefaultIndex sets the git index to our HEAD -func (t *TemporaryUploadRepository) SetDefaultIndex() error { - if _, _, err := git.NewCommand(t.ctx, "read-tree", "HEAD").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { +func (t *TemporaryUploadRepository) SetDefaultIndex(ctx context.Context) error { + if _, _, err := git.NewCommand("read-tree", "HEAD").RunStdString(ctx, &git.RunOpts{Dir: t.basePath}); err != nil { return fmt.Errorf("SetDefaultIndex: %w", err) } return nil } // RefreshIndex looks at the current index and checks to see if merges or updates are needed by checking stat() information. -func (t *TemporaryUploadRepository) RefreshIndex() error { - if _, _, err := git.NewCommand(t.ctx, "update-index", "--refresh").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { +func (t *TemporaryUploadRepository) RefreshIndex(ctx context.Context) error { + if _, _, err := git.NewCommand("update-index", "--refresh").RunStdString(ctx, &git.RunOpts{Dir: t.basePath}); err != nil { return fmt.Errorf("RefreshIndex: %w", err) } return nil } // LsFiles checks if the given filename arguments are in the index -func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, error) { +func (t *TemporaryUploadRepository) LsFiles(ctx context.Context, filenames ...string) ([]string, error) { stdOut := new(bytes.Buffer) stdErr := new(bytes.Buffer) - if err := git.NewCommand(t.ctx, "ls-files", "-z").AddDashesAndList(filenames...). - Run(&git.RunOpts{ + if err := git.NewCommand("ls-files", "-z").AddDashesAndList(filenames...). + Run(ctx, &git.RunOpts{ Dir: t.basePath, Stdout: stdOut, Stderr: stdErr, @@ -126,7 +128,7 @@ func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, erro } fileList := make([]string, 0, len(filenames)) - for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) { + for line := range bytes.SplitSeq(stdOut.Bytes(), []byte{'\000'}) { fileList = append(fileList, string(line)) } @@ -134,7 +136,7 @@ func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, erro } // RemoveFilesFromIndex removes the given files from the index -func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) error { +func (t *TemporaryUploadRepository) RemoveFilesFromIndex(ctx context.Context, filenames ...string) error { objFmt, err := t.gitRepo.GetObjectFormat() if err != nil { return fmt.Errorf("unable to get object format for temporary repo: %q, error: %w", t.repo.FullName(), err) @@ -150,8 +152,8 @@ func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) er } } - if err := git.NewCommand(t.ctx, "update-index", "--remove", "-z", "--index-info"). - Run(&git.RunOpts{ + if err := git.NewCommand("update-index", "--remove", "-z", "--index-info"). + Run(ctx, &git.RunOpts{ Dir: t.basePath, Stdin: stdIn, Stdout: stdOut, @@ -162,13 +164,13 @@ func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) er return nil } -// HashObject writes the provided content to the object db and returns its hash -func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) { +// HashObjectAndWrite writes the provided content to the object db and returns its hash +func (t *TemporaryUploadRepository) HashObjectAndWrite(ctx context.Context, content io.Reader) (string, error) { stdOut := new(bytes.Buffer) stdErr := new(bytes.Buffer) - if err := git.NewCommand(t.ctx, "hash-object", "-w", "--stdin"). - Run(&git.RunOpts{ + if err := git.NewCommand("hash-object", "-w", "--stdin"). + Run(ctx, &git.RunOpts{ Dir: t.basePath, Stdin: content, Stdout: stdOut, @@ -182,8 +184,8 @@ func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error } // AddObjectToIndex adds the provided object hash to the index with the provided mode and path -func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error { - if _, _, err := git.NewCommand(t.ctx, "update-index", "--add", "--replace", "--cacheinfo").AddDynamicArguments(mode, objectHash, objectPath).RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { +func (t *TemporaryUploadRepository) AddObjectToIndex(ctx context.Context, mode, objectHash, objectPath string) error { + if _, _, err := git.NewCommand("update-index", "--add", "--replace", "--cacheinfo").AddDynamicArguments(mode, objectHash, objectPath).RunStdString(ctx, &git.RunOpts{Dir: t.basePath}); err != nil { stderr := err.Error() if matched, _ := regexp.MatchString(".*Invalid path '.*", stderr); matched { return ErrFilePathInvalid{ @@ -198,8 +200,8 @@ func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPat } // WriteTree writes the current index as a tree to the object db and returns its hash -func (t *TemporaryUploadRepository) WriteTree() (string, error) { - stdout, _, err := git.NewCommand(t.ctx, "write-tree").RunStdString(&git.RunOpts{Dir: t.basePath}) +func (t *TemporaryUploadRepository) WriteTree(ctx context.Context) (string, error) { + stdout, _, err := git.NewCommand("write-tree").RunStdString(ctx, &git.RunOpts{Dir: t.basePath}) if err != nil { log.Error("Unable to write tree in temporary repo: %s(%s): Error: %v", t.repo.FullName(), t.basePath, err) return "", fmt.Errorf("Unable to write-tree in temporary repo for: %s Error: %w", t.repo.FullName(), err) @@ -208,16 +210,16 @@ func (t *TemporaryUploadRepository) WriteTree() (string, error) { } // GetLastCommit gets the last commit ID SHA of the repo -func (t *TemporaryUploadRepository) GetLastCommit() (string, error) { - return t.GetLastCommitByRef("HEAD") +func (t *TemporaryUploadRepository) GetLastCommit(ctx context.Context) (string, error) { + return t.GetLastCommitByRef(ctx, "HEAD") } // GetLastCommitByRef gets the last commit ID SHA of the repo by ref -func (t *TemporaryUploadRepository) GetLastCommitByRef(ref string) (string, error) { +func (t *TemporaryUploadRepository) GetLastCommitByRef(ctx context.Context, ref string) (string, error) { if ref == "" { ref = "HEAD" } - stdout, _, err := git.NewCommand(t.ctx, "rev-parse").AddDynamicArguments(ref).RunStdString(&git.RunOpts{Dir: t.basePath}) + stdout, _, err := git.NewCommand("rev-parse").AddDynamicArguments(ref).RunStdString(ctx, &git.RunOpts{Dir: t.basePath}) if err != nil { log.Error("Unable to get last ref for %s in temporary repo: %s(%s): Error: %v", ref, t.repo.FullName(), t.basePath, err) return "", fmt.Errorf("Unable to rev-parse %s in temporary repo for: %s Error: %w", ref, t.repo.FullName(), err) @@ -225,15 +227,53 @@ func (t *TemporaryUploadRepository) GetLastCommitByRef(ref string) (string, erro return strings.TrimSpace(stdout), nil } -// CommitTree creates a commit from a given tree for the user with provided message -func (t *TemporaryUploadRepository) CommitTree(parent string, author, committer *user_model.User, treeHash, message string, signoff bool) (string, error) { - return t.CommitTreeWithDate(parent, author, committer, treeHash, message, signoff, time.Now(), time.Now()) +type CommitTreeUserOptions struct { + ParentCommitID string + TreeHash string + CommitMessage string + SignOff bool + + DoerUser *user_model.User + + AuthorIdentity *IdentityOptions // if nil, use doer + AuthorTime *time.Time // if nil, use now + CommitterIdentity *IdentityOptions + CommitterTime *time.Time +} + +func makeGitUserSignature(doer *user_model.User, identity, other *IdentityOptions) *git.Signature { + gitSig := &git.Signature{} + if identity != nil { + gitSig.Name, gitSig.Email = identity.GitUserName, identity.GitUserEmail + } + if other != nil { + gitSig.Name = util.IfZero(gitSig.Name, other.GitUserName) + gitSig.Email = util.IfZero(gitSig.Email, other.GitUserEmail) + } + if gitSig.Name == "" { + gitSig.Name = doer.GitName() + } + if gitSig.Email == "" { + gitSig.Email = doer.GetEmail() + } + return gitSig } -// CommitTreeWithDate creates a commit from a given tree for the user with provided message -func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, committer *user_model.User, treeHash, message string, signoff bool, authorDate, committerDate time.Time) (string, error) { - authorSig := author.NewGitSig() - committerSig := committer.NewGitSig() +// CommitTree creates a commit from a given tree for the user with provided message +func (t *TemporaryUploadRepository) CommitTree(ctx context.Context, opts *CommitTreeUserOptions) (string, error) { + authorSig := makeGitUserSignature(opts.DoerUser, opts.AuthorIdentity, opts.CommitterIdentity) + committerSig := makeGitUserSignature(opts.DoerUser, opts.CommitterIdentity, opts.AuthorIdentity) + + authorDate := opts.AuthorTime + committerDate := opts.CommitterTime + if authorDate == nil && committerDate == nil { + authorDate = util.ToPointer(time.Now()) + committerDate = authorDate + } else if authorDate == nil { + authorDate = committerDate + } else if committerDate == nil { + committerDate = authorDate + } // Because this may call hooks we should pass in the environment env := append(os.Environ(), @@ -244,24 +284,27 @@ func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, co ) messageBytes := new(bytes.Buffer) - _, _ = messageBytes.WriteString(message) + _, _ = messageBytes.WriteString(opts.CommitMessage) _, _ = messageBytes.WriteString("\n") - cmdCommitTree := git.NewCommand(t.ctx, "commit-tree").AddDynamicArguments(treeHash) - if parent != "" { - cmdCommitTree.AddOptionValues("-p", parent) + cmdCommitTree := git.NewCommand("commit-tree").AddDynamicArguments(opts.TreeHash) + if opts.ParentCommitID != "" { + cmdCommitTree.AddOptionValues("-p", opts.ParentCommitID) } var sign bool - var keyID string + var key *git.SigningKey var signer *git.Signature - if parent != "" { - sign, keyID, signer, _ = asymkey_service.SignCRUDAction(t.ctx, t.repo.RepoPath(), author, t.basePath, parent) + if opts.ParentCommitID != "" { + sign, key, signer, _ = asymkey_service.SignCRUDAction(ctx, t.repo.RepoPath(), opts.DoerUser, t.basePath, opts.ParentCommitID) } else { - sign, keyID, signer, _ = asymkey_service.SignInitialCommit(t.ctx, t.repo.RepoPath(), author) + sign, key, signer, _ = asymkey_service.SignInitialCommit(ctx, t.repo.RepoPath(), opts.DoerUser) } if sign { - cmdCommitTree.AddOptionFormat("-S%s", keyID) + if key.Format != "" { + cmdCommitTree.AddConfig("gpg.format", key.Format) + } + cmdCommitTree.AddOptionFormat("-S%s", key.KeyID) if t.repo.GetTrustModel() == repo_model.CommitterTrustModel || t.repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { if committerSig.Name != authorSig.Name || committerSig.Email != authorSig.Email { // Add trailers @@ -279,7 +322,7 @@ func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, co cmdCommitTree.AddArguments("--no-gpg-sign") } - if signoff { + if opts.SignOff { // Signed-off-by _, _ = messageBytes.WriteString("\n") _, _ = messageBytes.WriteString("Signed-off-by: ") @@ -294,7 +337,7 @@ func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, co stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) if err := cmdCommitTree. - Run(&git.RunOpts{ + Run(ctx, &git.RunOpts{ Env: env, Dir: t.basePath, Stdin: messageBytes, @@ -310,10 +353,10 @@ func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, co } // Push the provided commitHash to the repository branch by the provided user -func (t *TemporaryUploadRepository) Push(doer *user_model.User, commitHash, branch string) error { +func (t *TemporaryUploadRepository) Push(ctx context.Context, doer *user_model.User, commitHash, branch string) error { // Because calls hooks we need to pass in the environment env := repo_module.PushingEnvironment(doer, t.repo) - if err := git.Push(t.ctx, t.basePath, git.PushOptions{ + if err := git.Push(ctx, t.basePath, git.PushOptions{ Remote: t.repo.RepoPath(), Branch: strings.TrimSpace(commitHash) + ":" + git.BranchPrefix + strings.TrimSpace(branch), Env: env, @@ -335,7 +378,7 @@ func (t *TemporaryUploadRepository) Push(doer *user_model.User, commitHash, bran } // DiffIndex returns a Diff of the current index to the head -func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) { +func (t *TemporaryUploadRepository) DiffIndex(ctx context.Context) (*gitdiff.Diff, error) { stdoutReader, stdoutWriter, err := os.Pipe() if err != nil { return nil, fmt.Errorf("unable to open stdout pipe: %w", err) @@ -346,8 +389,8 @@ func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) { }() stderr := new(bytes.Buffer) var diff *gitdiff.Diff - err = git.NewCommand(t.ctx, "diff-index", "--src-prefix=\\a/", "--dst-prefix=\\b/", "--cached", "-p", "HEAD"). - Run(&git.RunOpts{ + err = git.NewCommand("diff-index", "--src-prefix=\\a/", "--dst-prefix=\\b/", "--cached", "-p", "HEAD"). + Run(ctx, &git.RunOpts{ Timeout: 30 * time.Second, Dir: t.basePath, Stdout: stdoutWriter, @@ -356,7 +399,7 @@ func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) { _ = stdoutWriter.Close() defer cancel() var diffErr error - diff, diffErr = gitdiff.ParsePatch(t.ctx, setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdoutReader, "") + diff, diffErr = gitdiff.ParsePatch(ctx, setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdoutReader, "") _ = stdoutReader.Close() if diffErr != nil { // if the diffErr is not nil, it will be returned as the error of "Run()" @@ -370,18 +413,13 @@ func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) { return nil, fmt.Errorf("unable to run diff-index pipeline in temporary repo: %w", err) } - diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(t.ctx, t.basePath, git.TrustedCmdArgs{"--cached"}, "HEAD") - if err != nil { - return nil, err - } - return diff, nil } // GetBranchCommit Gets the commit object of the given branch func (t *TemporaryUploadRepository) GetBranchCommit(branch string) (*git.Commit, error) { if t.gitRepo == nil { - return nil, fmt.Errorf("repository has not been cloned") + return nil, errors.New("repository has not been cloned") } return t.gitRepo.GetBranchCommit(branch) } @@ -389,7 +427,7 @@ func (t *TemporaryUploadRepository) GetBranchCommit(branch string) (*git.Commit, // GetCommit Gets the commit object of the given commit ID func (t *TemporaryUploadRepository) GetCommit(commitID string) (*git.Commit, error) { if t.gitRepo == nil { - return nil, fmt.Errorf("repository has not been cloned") + return nil, errors.New("repository has not been cloned") } return t.gitRepo.GetCommit(commitID) } diff --git a/services/repository/files/tree.go b/services/repository/files/tree.go index 6775186afd..f2cbacbf1c 100644 --- a/services/repository/files/tree.go +++ b/services/repository/files/tree.go @@ -6,10 +6,16 @@ package files import ( "context" "fmt" + "html/template" "net/url" + "path" + "sort" + "strings" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/fileicon" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -88,11 +94,7 @@ func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git if len(entries) > perPage { tree.Truncated = true } - if rangeStart+perPage < len(entries) { - rangeEnd = rangeStart + perPage - } else { - rangeEnd = len(entries) - } + rangeEnd = min(rangeStart+perPage, len(entries)) tree.Entries = make([]api.GitEntry, rangeEnd-rangeStart) for e := rangeStart; e < rangeEnd; e++ { i := e - rangeStart @@ -118,3 +120,110 @@ func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git } return tree, nil } + +func entryModeString(entryMode git.EntryMode) string { + switch entryMode { + case git.EntryModeBlob: + return "blob" + case git.EntryModeExec: + return "exec" + case git.EntryModeSymlink: + return "symlink" + case git.EntryModeCommit: + return "commit" // submodule + case git.EntryModeTree: + return "tree" + } + return "unknown" +} + +type TreeViewNode struct { + EntryName string `json:"entryName"` + EntryMode string `json:"entryMode"` + EntryIcon template.HTML `json:"entryIcon"` + EntryIconOpen template.HTML `json:"entryIconOpen,omitempty"` + + SymLinkedToMode string `json:"symLinkedToMode,omitempty"` // TODO: for the EntryMode="symlink" + + FullPath string `json:"fullPath"` + SubmoduleURL string `json:"submoduleUrl,omitempty"` + Children []*TreeViewNode `json:"children,omitempty"` +} + +func (node *TreeViewNode) sortLevel() int { + return util.Iif(node.EntryMode == "tree" || node.EntryMode == "commit", 0, 1) +} + +func newTreeViewNodeFromEntry(ctx context.Context, renderedIconPool *fileicon.RenderedIconPool, commit *git.Commit, parentDir string, entry *git.TreeEntry) *TreeViewNode { + node := &TreeViewNode{ + EntryName: entry.Name(), + EntryMode: entryModeString(entry.Mode()), + FullPath: path.Join(parentDir, entry.Name()), + } + + entryInfo := fileicon.EntryInfoFromGitTreeEntry(commit, node.FullPath, entry) + node.EntryIcon = fileicon.RenderEntryIconHTML(renderedIconPool, entryInfo) + if entryInfo.EntryMode.IsDir() { + entryInfo.IsOpen = true + node.EntryIconOpen = fileicon.RenderEntryIconHTML(renderedIconPool, entryInfo) + } + + if node.EntryMode == "commit" { + if subModule, err := commit.GetSubModule(node.FullPath); err != nil { + log.Error("GetSubModule: %v", err) + } else if subModule != nil { + submoduleFile := git.NewCommitSubmoduleFile(subModule.URL, entry.ID.String()) + webLink := submoduleFile.SubmoduleWebLink(ctx) + node.SubmoduleURL = webLink.CommitWebLink + } + } + + return node +} + +// sortTreeViewNodes list directory first and with alpha sequence +func sortTreeViewNodes(nodes []*TreeViewNode) { + sort.Slice(nodes, func(i, j int) bool { + a, b := nodes[i].sortLevel(), nodes[j].sortLevel() + if a != b { + return a < b + } + return nodes[i].EntryName < nodes[j].EntryName + }) +} + +func listTreeNodes(ctx context.Context, renderedIconPool *fileicon.RenderedIconPool, commit *git.Commit, tree *git.Tree, treePath, subPath string) ([]*TreeViewNode, error) { + entries, err := tree.ListEntries() + if err != nil { + return nil, err + } + + subPathDirName, subPathRemaining, _ := strings.Cut(subPath, "/") + nodes := make([]*TreeViewNode, 0, len(entries)) + for _, entry := range entries { + node := newTreeViewNodeFromEntry(ctx, renderedIconPool, commit, treePath, entry) + nodes = append(nodes, node) + if entry.IsDir() && subPathDirName == entry.Name() { + subTreePath := treePath + "/" + node.EntryName + if subTreePath[0] == '/' { + subTreePath = subTreePath[1:] + } + subNodes, err := listTreeNodes(ctx, renderedIconPool, commit, entry.Tree(), subTreePath, subPathRemaining) + if err != nil { + log.Error("listTreeNodes: %v", err) + } else { + node.Children = subNodes + } + } + } + sortTreeViewNodes(nodes) + return nodes, nil +} + +func GetTreeViewNodes(ctx context.Context, renderedIconPool *fileicon.RenderedIconPool, commit *git.Commit, treePath, subPath string) ([]*TreeViewNode, error) { + entry, err := commit.GetTreeEntryByPath(treePath) + if err != nil { + return nil, err + } + return listTreeNodes(ctx, renderedIconPool, commit, entry.Tree(), treePath, subPath) +} diff --git a/services/repository/files/tree_test.go b/services/repository/files/tree_test.go index 0c60fddf7b..a53f342d40 100644 --- a/services/repository/files/tree_test.go +++ b/services/repository/files/tree_test.go @@ -4,9 +4,12 @@ package files import ( + "html/template" "testing" "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/fileicon" + "code.gitea.io/gitea/modules/git" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/services/contexttest" @@ -48,5 +51,69 @@ func TestGetTreeBySHA(t *testing.T) { TotalCount: 1, } - assert.EqualValues(t, expectedTree, tree) + assert.Equal(t, expectedTree, tree) +} + +func TestGetTreeViewNodes(t *testing.T) { + unittest.PrepareTestEnv(t) + ctx, _ := contexttest.MockContext(t, "user2/repo1") + ctx.Repo.RefFullName = git.RefNameFromBranch("sub-home-md-img-check") + contexttest.LoadRepo(t, ctx, 1) + contexttest.LoadRepoCommit(t, ctx) + contexttest.LoadUser(t, ctx, 2) + contexttest.LoadGitRepo(t, ctx) + defer ctx.Repo.GitRepo.Close() + + renderedIconPool := fileicon.NewRenderedIconPool() + mockIconForFile := func(id string) template.HTML { + return template.HTML(`<svg class="svg git-entry-icon octicon-file" width="16" height="16" aria-hidden="true"><use xlink:href="#` + id + `"></use></svg>`) + } + mockIconForFolder := func(id string) template.HTML { + return template.HTML(`<svg class="svg git-entry-icon octicon-file-directory-fill" width="16" height="16" aria-hidden="true"><use xlink:href="#` + id + `"></use></svg>`) + } + mockOpenIconForFolder := func(id string) template.HTML { + return template.HTML(`<svg class="svg git-entry-icon octicon-file-directory-open-fill" width="16" height="16" aria-hidden="true"><use xlink:href="#` + id + `"></use></svg>`) + } + treeNodes, err := GetTreeViewNodes(ctx, renderedIconPool, ctx.Repo.Commit, "", "") + assert.NoError(t, err) + assert.Equal(t, []*TreeViewNode{ + { + EntryName: "docs", + EntryMode: "tree", + FullPath: "docs", + EntryIcon: mockIconForFolder(`svg-mfi-folder-docs`), + EntryIconOpen: mockOpenIconForFolder(`svg-mfi-folder-docs`), + }, + }, treeNodes) + + treeNodes, err = GetTreeViewNodes(ctx, renderedIconPool, ctx.Repo.Commit, "", "docs/README.md") + assert.NoError(t, err) + assert.Equal(t, []*TreeViewNode{ + { + EntryName: "docs", + EntryMode: "tree", + FullPath: "docs", + EntryIcon: mockIconForFolder(`svg-mfi-folder-docs`), + EntryIconOpen: mockOpenIconForFolder(`svg-mfi-folder-docs`), + Children: []*TreeViewNode{ + { + EntryName: "README.md", + EntryMode: "blob", + FullPath: "docs/README.md", + EntryIcon: mockIconForFile(`svg-mfi-readme`), + }, + }, + }, + }, treeNodes) + + treeNodes, err = GetTreeViewNodes(ctx, renderedIconPool, ctx.Repo.Commit, "docs", "README.md") + assert.NoError(t, err) + assert.Equal(t, []*TreeViewNode{ + { + EntryName: "README.md", + EntryMode: "blob", + FullPath: "docs/README.md", + EntryIcon: mockIconForFile(`svg-mfi-readme`), + }, + }, treeNodes) } diff --git a/services/repository/files/update.go b/services/repository/files/update.go index a2763105b0..e871f777e5 100644 --- a/services/repository/files/update.go +++ b/services/repository/files/update.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "path" + "slices" "strings" "time" @@ -15,20 +16,22 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" asymkey_service "code.gitea.io/gitea/services/asymkey" pull_service "code.gitea.io/gitea/services/pull" ) // IdentityOptions for a person's identity like an author or committer type IdentityOptions struct { - Name string - Email string + GitUserName string // to match "git config user.name" + GitUserEmail string // to match "git config user.email" } // CommitDateOptions store dates for GIT_AUTHOR_DATE and GIT_COMMITTER_DATE @@ -85,14 +88,32 @@ func (err ErrRepoFileDoesNotExist) Unwrap() error { return util.ErrNotExist } +type LazyReadSeeker interface { + io.ReadSeeker + io.Closer + OpenLazyReader() error +} + // ChangeRepoFiles adds, updates or removes multiple files in the given repository -func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ChangeRepoFilesOptions) (*structs.FilesResponse, error) { +func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ChangeRepoFilesOptions) (_ *structs.FilesResponse, errRet error) { + var addedLfsPointers []lfs.Pointer + defer func() { + if errRet != nil { + for _, lfsPointer := range addedLfsPointers { + _, err := git_model.RemoveLFSMetaObjectByOid(ctx, repo.ID, lfsPointer.Oid) + if err != nil { + log.Error("ChangeRepoFiles: RemoveLFSMetaObjectByOid failed: %v", err) + } + } + } + }() + err := repo.MustNotBeArchived() if err != nil { return nil, err } - // If no branch name is set, assume default branch + // If no branch name is set, assume the default branch if opts.OldBranch == "" { opts.OldBranch = repo.DefaultBranch } @@ -107,8 +128,13 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use defer closer.Close() // oldBranch must exist for this operation - if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil && !repo.IsEmpty { + if exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.OldBranch); err != nil { return nil, err + } else if !exist && !repo.IsEmpty { + return nil, git_model.ErrBranchNotExist{ + RepoID: repo.ID, + BranchName: opts.OldBranch, + } } var treePaths []string @@ -119,14 +145,14 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } // Check that the path given in opts.treePath is valid (not a git path) - treePath := CleanUploadFileName(file.TreePath) + treePath := CleanGitTreePath(file.TreePath) if treePath == "" { return nil, ErrFilenameInvalid{ Path: file.TreePath, } } // If there is a fromTreePath (we are copying it), also clean it up - fromTreePath := CleanUploadFileName(file.FromTreePath) + fromTreePath := CleanGitTreePath(file.FromTreePath) if fromTreePath == "" && file.FromTreePath != "" { return nil, ErrFilenameInvalid{ Path: file.FromTreePath, @@ -145,30 +171,28 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use // Check to make sure the branch does not already exist, otherwise we can't proceed. // If we aren't branching to a new branch, make sure user can commit to the given branch if opts.NewBranch != opts.OldBranch { - existingBranch, err := gitRepo.GetBranch(opts.NewBranch) - if existingBranch != nil { + exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.NewBranch) + if err != nil { + return nil, err + } + if exist { return nil, git_model.ErrBranchAlreadyExists{ BranchName: opts.NewBranch, } } - if err != nil && !git.IsErrBranchNotExist(err) { - return nil, err - } } else if err := VerifyBranchProtection(ctx, repo, doer, opts.OldBranch, treePaths); err != nil { return nil, err } message := strings.TrimSpace(opts.Message) - author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) - - t, err := NewTemporaryUploadRepository(ctx, repo) + t, err := NewTemporaryUploadRepository(repo) if err != nil { log.Error("NewTemporaryUploadRepository failed: %v", err) } defer t.Close() hasOldBranch := true - if err := t.Clone(opts.OldBranch, true); err != nil { + if err := t.Clone(ctx, opts.OldBranch, true); err != nil { for _, file := range opts.Files { if file.Operation == "delete" { return nil, err @@ -177,14 +201,14 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { return nil, err } - if err := t.Init(repo.ObjectFormatName); err != nil { + if err := t.Init(ctx, repo.ObjectFormatName); err != nil { return nil, err } hasOldBranch = false opts.LastCommitID = "" } if hasOldBranch { - if err := t.SetDefaultIndex(); err != nil { + if err := t.SetDefaultIndex(ctx); err != nil { return nil, err } } @@ -192,19 +216,13 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use for _, file := range opts.Files { if file.Operation == "delete" { // Get the files in the index - filesInIndex, err := t.LsFiles(file.TreePath) + filesInIndex, err := t.LsFiles(ctx, file.TreePath) if err != nil { return nil, fmt.Errorf("DeleteRepoFile: %w", err) } // Find the file we want to delete in the index - inFilelist := false - for _, indexFile := range filesInIndex { - if indexFile == file.TreePath { - inFilelist = true - break - } - } + inFilelist := slices.Contains(filesInIndex, file.TreePath) if !inFilelist { return nil, ErrRepoFileDoesNotExist{ Path: file.TreePath, @@ -220,7 +238,7 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use return nil, err // Couldn't get a commit for the branch } - // Assigned LastCommitID in opts if it hasn't been set + // Assigned LastCommitID in "opts" if it hasn't been set if opts.LastCommitID == "" { opts.LastCommitID = commit.ID.String() } else { @@ -232,22 +250,25 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } for _, file := range opts.Files { - if err := handleCheckErrors(file, commit, opts); err != nil { + if err = handleCheckErrors(file, commit, opts); err != nil { return nil, err } } } - contentStore := lfs.NewContentStore() + lfsContentStore := lfs.NewContentStore() for _, file := range opts.Files { switch file.Operation { - case "create", "update": - if err := CreateOrUpdateFile(ctx, t, file, contentStore, repo.ID, hasOldBranch); err != nil { + case "create", "update", "rename", "upload": + addedLfsPointer, err := modifyFile(ctx, t, file, lfsContentStore, repo.ID) + if err != nil { return nil, err } + if addedLfsPointer != nil { + addedLfsPointers = append(addedLfsPointers, *addedLfsPointer) + } case "delete": - // Remove the file from the index - if err := t.RemoveFilesFromIndex(file.TreePath); err != nil { + if err = t.RemoveFilesFromIndex(ctx, file.TreePath); err != nil { return nil, err } default: @@ -256,24 +277,33 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } // Now write the tree - treeHash, err := t.WriteTree() + treeHash, err := t.WriteTree(ctx) if err != nil { return nil, err } // Now commit the tree - var commitHash string + commitOpts := &CommitTreeUserOptions{ + ParentCommitID: opts.LastCommitID, + TreeHash: treeHash, + CommitMessage: message, + SignOff: opts.Signoff, + DoerUser: doer, + AuthorIdentity: opts.Author, + AuthorTime: nil, + CommitterIdentity: opts.Committer, + CommitterTime: nil, + } if opts.Dates != nil { - commitHash, err = t.CommitTreeWithDate(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) - } else { - commitHash, err = t.CommitTree(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff) + commitOpts.AuthorTime, commitOpts.CommitterTime = &opts.Dates.Author, &opts.Dates.Committer } + commitHash, err := t.CommitTree(ctx, commitOpts) if err != nil { return nil, err } // Then push this tree to NewBranch - if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + if err := t.Push(ctx, doer, commitHash, opts.NewBranch); err != nil { log.Error("%T %v", err, err) return nil, err } @@ -283,14 +313,16 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use return nil, err } - filesResponse, err := GetFilesResponseFromCommit(ctx, repo, commit, opts.NewBranch, treePaths) + // FIXME: this call seems not right, why it needs to read the file content again + // FIXME: why it uses the NewBranch as "ref", it should use the commit ID because the response is only for this commit + filesResponse, err := GetFilesResponseFromCommit(ctx, repo, gitRepo, utils.NewRefCommit(git.RefNameFromBranch(opts.NewBranch), commit), treePaths) if err != nil { return nil, err } if repo.IsEmpty { if isEmpty, err := gitRepo.IsEmpty(); err == nil && !isEmpty { - _ = repo_model.UpdateRepositoryCols(ctx, &repo_model.Repository{ID: repo.ID, IsEmpty: false, DefaultBranch: opts.NewBranch}, "is_empty", "default_branch") + _ = repo_model.UpdateRepositoryColsWithAutoTime(ctx, &repo_model.Repository{ID: repo.ID, IsEmpty: false, DefaultBranch: opts.NewBranch}, "is_empty", "default_branch") } } @@ -356,22 +388,33 @@ func (err ErrSHAOrCommitIDNotProvided) Error() string { // handles the check for various issues for ChangeRepoFiles func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRepoFilesOptions) error { - if file.Operation == "update" || file.Operation == "delete" { - fromEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) - if err != nil { - return err + // check old entry (fromTreePath/fromEntry) + if file.Operation == "update" || file.Operation == "upload" || file.Operation == "delete" || file.Operation == "rename" { + var fromEntryIDString string + { + fromEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) + if file.Operation == "upload" && git.IsErrNotExist(err) { + fromEntry = nil + } else if err != nil { + return err + } + if fromEntry != nil { + fromEntryIDString = fromEntry.ID.String() + file.Options.executable = fromEntry.IsExecutable() // FIXME: legacy hacky approach, it shouldn't prepare the "Options" in the "check" function + } } + if file.SHA != "" { - // If a SHA was given and the SHA given doesn't match the SHA of the fromTreePath, throw error - if file.SHA != fromEntry.ID.String() { + // If the SHA given doesn't match the SHA of the fromTreePath, throw error + if file.SHA != fromEntryIDString { return pull_service.ErrSHADoesNotMatch{ Path: file.Options.treePath, GivenSHA: file.SHA, - CurrentSHA: fromEntry.ID.String(), + CurrentSHA: fromEntryIDString, } } } else if opts.LastCommitID != "" { - // If a lastCommitID was given and it doesn't match the commitID of the head of the branch throw + // If a lastCommitID given doesn't match the branch head's commitID throw // an error, but only if we aren't creating a new branch. if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch { if changed, err := commit.FileChangedSinceCommit(file.Options.treePath, opts.LastCommitID); err != nil { @@ -389,13 +432,13 @@ func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRep // haven't been made. We throw an error if one wasn't provided. return ErrSHAOrCommitIDNotProvided{} } - file.Options.executable = fromEntry.IsExecutable() } - if file.Operation == "create" || file.Operation == "update" { - // For the path where this file will be created/updated, we need to make - // sure no parts of the path are existing files or links except for the last - // item in the path which is the file name, and that shouldn't exist IF it is - // a new file OR is being moved to a new path. + + // check new entry (treePath/treeEntry) + if file.Operation == "create" || file.Operation == "update" || file.Operation == "upload" || file.Operation == "rename" { + // For operation's target path, we need to make sure no parts of the path are existing files or links + // except for the last item in the path (which is the file name). + // And that shouldn't exist IF it is a new file OR is being moved to a new path. treePathParts := strings.Split(file.Options.treePath, "/") subTreePath := "" for index, part := range treePathParts { @@ -432,7 +475,7 @@ func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRep Type: git.EntryModeTree, } } else if file.Options.fromTreePath != file.Options.treePath || file.Operation == "create" { - // The entry shouldn't exist if we are creating new file or moving to a new path + // The entry shouldn't exist if we are creating the new file or moving to a new path return ErrRepoFileAlreadyExists{ Path: file.Options.treePath, } @@ -443,21 +486,23 @@ func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRep return nil } -// CreateOrUpdateFile handles creating or updating a file for ChangeRepoFiles -func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile, contentStore *lfs.ContentStore, repoID int64, hasOldBranch bool) error { +func modifyFile(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile, contentStore *lfs.ContentStore, repoID int64) (addedLfsPointer *lfs.Pointer, _ error) { + if rd, ok := file.ContentReader.(LazyReadSeeker); ok { + if err := rd.OpenLazyReader(); err != nil { + return nil, fmt.Errorf("OpenLazyReader: %w", err) + } + defer rd.Close() + } + // Get the two paths (might be the same if not moving) from the index if they exist - filesInIndex, err := t.LsFiles(file.TreePath, file.FromTreePath) + filesInIndex, err := t.LsFiles(ctx, file.TreePath, file.FromTreePath) if err != nil { - return fmt.Errorf("UpdateRepoFile: %w", err) + return nil, fmt.Errorf("LsFiles: %w", err) } // If is a new file (not updating) then the given path shouldn't exist if file.Operation == "create" { - for _, indexFile := range filesInIndex { - if indexFile == file.TreePath { - return ErrRepoFileAlreadyExists{ - Path: file.TreePath, - } - } + if slices.Contains(filesInIndex, file.TreePath) { + return nil, ErrRepoFileAlreadyExists{Path: file.TreePath} } } @@ -465,79 +510,178 @@ func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file if file.Options.fromTreePath != file.Options.treePath && len(filesInIndex) > 0 { for _, indexFile := range filesInIndex { if indexFile == file.Options.fromTreePath { - if err := t.RemoveFilesFromIndex(file.FromTreePath); err != nil { - return err + if err = t.RemoveFilesFromIndex(ctx, file.FromTreePath); err != nil { + return nil, err } } } } - treeObjectContentReader := file.ContentReader - var lfsMetaObject *git_model.LFSMetaObject - if setting.LFS.StartServer && hasOldBranch { - // Check there is no way this can return multiple infos - filename2attribute2info, err := t.gitRepo.CheckAttribute(git.CheckAttributeOpts{ - Attributes: []string{"filter"}, - Filenames: []string{file.Options.treePath}, - CachedOnly: true, - }) + var writeObjectRet *writeRepoObjectRet + switch file.Operation { + case "create", "update", "upload": + writeObjectRet, err = writeRepoObjectForModify(ctx, t, file) + case "rename": + writeObjectRet, err = writeRepoObjectForRename(ctx, t, file) + default: + return nil, util.NewInvalidArgumentErrorf("unknown file modification operation: '%s'", file.Operation) + } + if err != nil { + return nil, err + } + + // Add the object to the index, the "file.Options.executable" is set in handleCheckErrors by the caller (legacy hacky approach) + if err = t.AddObjectToIndex(ctx, util.Iif(file.Options.executable, "100755", "100644"), writeObjectRet.ObjectHash, file.Options.treePath); err != nil { + return nil, err + } + + if writeObjectRet.LfsContent == nil { + return nil, nil // No LFS pointer, so nothing to do + } + defer writeObjectRet.LfsContent.Close() + + // Now we must store the content into an LFS object + lfsMetaObject, err := git_model.NewLFSMetaObject(ctx, repoID, writeObjectRet.LfsPointer) + if err != nil { + return nil, err + } + exist, err := contentStore.Exists(lfsMetaObject.Pointer) + if err != nil { + return nil, err + } + if !exist { + err = contentStore.Put(lfsMetaObject.Pointer, writeObjectRet.LfsContent) if err != nil { - return err + if _, errRemove := git_model.RemoveLFSMetaObjectByOid(ctx, repoID, lfsMetaObject.Oid); errRemove != nil { + return nil, fmt.Errorf("unable to remove failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, errRemove, err) + } + return nil, err } + } + return &lfsMetaObject.Pointer, nil +} - if filename2attribute2info[file.Options.treePath] != nil && filename2attribute2info[file.Options.treePath]["filter"] == "lfs" { - // OK so we are supposed to LFS this data! - pointer, err := lfs.GeneratePointer(treeObjectContentReader) +func checkIsLfsFileInGitAttributes(ctx context.Context, t *TemporaryUploadRepository, paths []string) (ret []bool, err error) { + attributesMap, err := attribute.CheckAttributes(ctx, t.gitRepo, "" /* use temp repo's working dir */, attribute.CheckAttributeOpts{ + Attributes: []string{attribute.Filter}, + Filenames: paths, + }) + if err != nil { + return nil, err + } + for _, p := range paths { + isLFSFile := attributesMap[p] != nil && attributesMap[p].Get(attribute.Filter).ToString().Value() == "lfs" + ret = append(ret, isLFSFile) + } + return ret, nil +} + +type writeRepoObjectRet struct { + ObjectHash string + LfsContent io.ReadCloser // if not nil, then the caller should store its content in LfsPointer, then close it + LfsPointer lfs.Pointer +} + +// writeRepoObjectForModify hashes the git object for create or update operations +func writeRepoObjectForModify(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile) (ret *writeRepoObjectRet, err error) { + ret = &writeRepoObjectRet{} + treeObjectContentReader := file.ContentReader + if setting.LFS.StartServer { + checkIsLfsFiles, err := checkIsLfsFileInGitAttributes(ctx, t, []string{file.Options.treePath}) + if err != nil { + return nil, err + } + if checkIsLfsFiles[0] { + // OK, so we are supposed to LFS this data! + ret.LfsPointer, err = lfs.GeneratePointer(file.ContentReader) if err != nil { - return err + return nil, err } - lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repoID} - treeObjectContentReader = strings.NewReader(pointer.StringContent()) + if _, err = file.ContentReader.Seek(0, io.SeekStart); err != nil { + return nil, err + } + ret.LfsContent = io.NopCloser(file.ContentReader) + treeObjectContentReader = strings.NewReader(ret.LfsPointer.StringContent()) } } - // Add the object to the database - objectHash, err := t.HashObject(treeObjectContentReader) + ret.ObjectHash, err = t.HashObjectAndWrite(ctx, treeObjectContentReader) if err != nil { - return err + return nil, err } + return ret, nil +} - // Add the object to the index - if file.Options.executable { - if err := t.AddObjectToIndex("100755", objectHash, file.Options.treePath); err != nil { - return err - } - } else { - if err := t.AddObjectToIndex("100644", objectHash, file.Options.treePath); err != nil { - return err +// writeRepoObjectForRename the same as writeRepoObjectForModify buf for "rename" +func writeRepoObjectForRename(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile) (ret *writeRepoObjectRet, err error) { + lastCommitID, err := t.GetLastCommit(ctx) + if err != nil { + return nil, err + } + commit, err := t.GetCommit(lastCommitID) + if err != nil { + return nil, err + } + oldEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) + if err != nil { + return nil, err + } + + ret = &writeRepoObjectRet{ObjectHash: oldEntry.ID.String()} + if !setting.LFS.StartServer { + return ret, nil + } + + checkIsLfsFiles, err := checkIsLfsFileInGitAttributes(ctx, t, []string{file.Options.fromTreePath, file.Options.treePath}) + if err != nil { + return nil, err + } + oldIsLfs, newIsLfs := checkIsLfsFiles[0], checkIsLfsFiles[1] + + // If the old and new paths are both in lfs or both not in lfs, the object hash of the old file can be used directly + // as the object doesn't change + if oldIsLfs == newIsLfs { + return ret, nil + } + + oldEntryBlobPointerBy := func(f func(r io.Reader) (lfs.Pointer, error)) (lfsPointer lfs.Pointer, err error) { + r, err := oldEntry.Blob().DataAsync() + if err != nil { + return lfsPointer, err } + defer r.Close() + return f(r) } - if lfsMetaObject != nil { - // We have an LFS object - create it - lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, lfsMetaObject.RepositoryID, lfsMetaObject.Pointer) + var treeObjectContentReader io.ReadCloser + if oldIsLfs { + // If the old is in lfs but the new isn't, read the content from lfs and add it as a normal git object + pointer, err := oldEntryBlobPointerBy(lfs.ReadPointer) if err != nil { - return err + return nil, err } - exist, err := contentStore.Exists(lfsMetaObject.Pointer) + treeObjectContentReader, err = lfs.ReadMetaObject(pointer) if err != nil { - return err + return nil, err } - if !exist { - _, err := file.ContentReader.Seek(0, io.SeekStart) - if err != nil { - return err - } - if err := contentStore.Put(lfsMetaObject.Pointer, file.ContentReader); err != nil { - if _, err2 := git_model.RemoveLFSMetaObjectByOid(ctx, repoID, lfsMetaObject.Oid); err2 != nil { - return fmt.Errorf("unable to remove failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, err2, err) - } - return err - } + defer treeObjectContentReader.Close() + } else { + // If the new is in lfs but the old isn't, read the content from the git object and generate a lfs pointer of it + ret.LfsPointer, err = oldEntryBlobPointerBy(lfs.GeneratePointer) + if err != nil { + return nil, err + } + ret.LfsContent, err = oldEntry.Blob().DataAsync() + if err != nil { + return nil, err } + treeObjectContentReader = io.NopCloser(strings.NewReader(ret.LfsPointer.StringContent())) } - - return nil + ret.ObjectHash, err = t.HashObjectAndWrite(ctx, treeObjectContentReader) + if err != nil { + return nil, err + } + return ret, nil } // VerifyBranchProtection verify the branch protection for modifying the given treePath on the given branch diff --git a/services/repository/files/upload.go b/services/repository/files/upload.go index cbfaf49d13..b783cbd01d 100644 --- a/services/repository/files/upload.go +++ b/services/repository/files/upload.go @@ -8,14 +8,11 @@ import ( "fmt" "os" "path" - "strings" + "sync" - git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/lfs" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/log" ) // UploadRepoFileOptions contains the uploaded repository file options @@ -27,199 +24,88 @@ type UploadRepoFileOptions struct { Message string Files []string // In UUID format. Signoff bool + Author *IdentityOptions + Committer *IdentityOptions } -type uploadInfo struct { - upload *repo_model.Upload - lfsMetaObject *git_model.LFSMetaObject +type lazyLocalFileReader struct { + *os.File + localFilename string + counter int + mu sync.Mutex } -func cleanUpAfterFailure(ctx context.Context, infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error { - for _, info := range *infos { - if info.lfsMetaObject == nil { - continue - } - if !info.lfsMetaObject.Existing { - if _, err := git_model.RemoveLFSMetaObjectByOid(ctx, t.repo.ID, info.lfsMetaObject.Oid); err != nil { - original = fmt.Errorf("%w, %v", original, err) // We wrap the original error - as this is the underlying error that required the fallback - } - } - } - return original -} - -// UploadRepoFiles uploads files to the given repository -func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *UploadRepoFileOptions) error { - if len(opts.Files) == 0 { - return nil - } +var _ LazyReadSeeker = (*lazyLocalFileReader)(nil) - uploads, err := repo_model.GetUploadsByUUIDs(ctx, opts.Files) - if err != nil { - return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %w", opts.Files, err) - } +func (l *lazyLocalFileReader) Close() error { + l.mu.Lock() + defer l.mu.Unlock() - names := make([]string, len(uploads)) - infos := make([]uploadInfo, len(uploads)) - for i, upload := range uploads { - // Check file is not lfs locked, will return nil if lock setting not enabled - filepath := path.Join(opts.TreePath, upload.Name) - lfsLock, err := git_model.GetTreePathLock(ctx, repo.ID, filepath) - if err != nil { - return err - } - if lfsLock != nil && lfsLock.OwnerID != doer.ID { - u, err := user_model.GetUserByID(ctx, lfsLock.OwnerID) - if err != nil { - return err + if l.counter > 0 { + l.counter-- + if l.counter == 0 { + if err := l.File.Close(); err != nil { + return fmt.Errorf("close file %s: %w", l.localFilename, err) } - return git_model.ErrLFSFileLocked{RepoID: repo.ID, Path: filepath, UserName: u.Name} - } - - names[i] = upload.Name - infos[i] = uploadInfo{upload: upload} - } - - t, err := NewTemporaryUploadRepository(ctx, repo) - if err != nil { - return err - } - defer t.Close() - - hasOldBranch := true - if err = t.Clone(opts.OldBranch, true); err != nil { - if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { - return err - } - if err = t.Init(repo.ObjectFormatName); err != nil { - return err - } - hasOldBranch = false - opts.LastCommitID = "" - } - if hasOldBranch { - if err = t.SetDefaultIndex(); err != nil { - return err - } - } - - var filename2attribute2info map[string]map[string]string - if setting.LFS.StartServer { - filename2attribute2info, err = t.gitRepo.CheckAttribute(git.CheckAttributeOpts{ - Attributes: []string{"filter"}, - Filenames: names, - CachedOnly: true, - }) - if err != nil { - return err + l.File = nil } + return nil } + return fmt.Errorf("file %s already closed", l.localFilename) +} - // Copy uploaded files into repository. - for i := range infos { - if err := copyUploadedLFSFileIntoRepository(&infos[i], filename2attribute2info, t, opts.TreePath); err != nil { - return err - } - } +func (l *lazyLocalFileReader) OpenLazyReader() error { + l.mu.Lock() + defer l.mu.Unlock() - // Now write the tree - treeHash, err := t.WriteTree() - if err != nil { - return err + if l.File != nil { + l.counter++ + return nil } - // make author and committer the doer - author := doer - committer := doer - - // Now commit the tree - commitHash, err := t.CommitTree(opts.LastCommitID, author, committer, treeHash, opts.Message, opts.Signoff) + file, err := os.Open(l.localFilename) if err != nil { return err } + l.File = file + l.counter = 1 + return nil +} - // Now deal with LFS objects - for i := range infos { - if infos[i].lfsMetaObject == nil { - continue - } - infos[i].lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, infos[i].lfsMetaObject.RepositoryID, infos[i].lfsMetaObject.Pointer) - if err != nil { - // OK Now we need to cleanup - return cleanUpAfterFailure(ctx, &infos, t, err) - } - // Don't move the files yet - we need to ensure that - // everything can be inserted first - } - - // OK now we can insert the data into the store - there's no way to clean up the store - // once it's in there, it's in there. - contentStore := lfs.NewContentStore() - for _, info := range infos { - if err := uploadToLFSContentStore(info, contentStore); err != nil { - return cleanUpAfterFailure(ctx, &infos, t, err) - } - } - - // Then push this tree to NewBranch - if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { - return err +// UploadRepoFiles uploads files to the given repository +func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *UploadRepoFileOptions) error { + if len(opts.Files) == 0 { + return nil } - return repo_model.DeleteUploads(ctx, uploads...) -} - -func copyUploadedLFSFileIntoRepository(info *uploadInfo, filename2attribute2info map[string]map[string]string, t *TemporaryUploadRepository, treePath string) error { - file, err := os.Open(info.upload.LocalPath()) + uploads, err := repo_model.GetUploadsByUUIDs(ctx, opts.Files) if err != nil { - return err + return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %w", opts.Files, err) } - defer file.Close() - var objectHash string - if setting.LFS.StartServer && filename2attribute2info[info.upload.Name] != nil && filename2attribute2info[info.upload.Name]["filter"] == "lfs" { - // Handle LFS - // FIXME: Inefficient! this should probably happen in models.Upload - pointer, err := lfs.GeneratePointer(file) - if err != nil { - return err - } - - info.lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: t.repo.ID} - - if objectHash, err = t.HashObject(strings.NewReader(pointer.StringContent())); err != nil { - return err - } - } else if objectHash, err = t.HashObject(file); err != nil { - return err + changeOpts := &ChangeRepoFilesOptions{ + LastCommitID: opts.LastCommitID, + OldBranch: opts.OldBranch, + NewBranch: opts.NewBranch, + Message: opts.Message, + Signoff: opts.Signoff, + Author: opts.Author, + Committer: opts.Committer, + } + for _, upload := range uploads { + changeOpts.Files = append(changeOpts.Files, &ChangeRepoFile{ + Operation: "upload", + TreePath: path.Join(opts.TreePath, upload.Name), + ContentReader: &lazyLocalFileReader{localFilename: upload.LocalPath()}, + }) } - // Add the object to the index - return t.AddObjectToIndex("100644", objectHash, path.Join(treePath, info.upload.Name)) -} - -func uploadToLFSContentStore(info uploadInfo, contentStore *lfs.ContentStore) error { - if info.lfsMetaObject == nil { - return nil - } - exist, err := contentStore.Exists(info.lfsMetaObject.Pointer) + _, err = ChangeRepoFiles(ctx, repo, doer, changeOpts) if err != nil { return err } - if !exist { - file, err := os.Open(info.upload.LocalPath()) - if err != nil { - return err - } - - defer file.Close() - // FIXME: Put regenerates the hash and copies the file over. - // I guess this strictly ensures the soundness of the store but this is inefficient. - if err := contentStore.Put(info.lfsMetaObject.Pointer, file); err != nil { - // OK Now we need to cleanup - // Can't clean up the store, once uploaded there they're there. - return err - } + if err := repo_model.DeleteUploads(ctx, uploads...); err != nil { + log.Error("DeleteUploads: %v", err) } return nil } diff --git a/services/repository/fork.go b/services/repository/fork.go index cff0b1a403..8bd3498b17 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -65,7 +65,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork } // Fork is prohibited, if user has reached maximum limit of repositories - if !owner.CanForkRepo() { + if !doer.CanForkRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ Limit: owner.MaxRepoCreation, } @@ -100,117 +100,106 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork IsFork: true, ForkID: opts.BaseRepo.ID, ObjectFormatName: opts.BaseRepo.ObjectFormatName, + Status: repo_model.RepositoryBeingMigrated, } - oldRepoPath := opts.BaseRepo.RepoPath() - - needsRollback := false - rollbackFn := func() { - if !needsRollback { - return - } - - repoPath := repo_model.RepoPath(owner.Name, repo.Name) - - if exists, _ := util.IsExist(repoPath); !exists { - return - } - - // As the transaction will be failed and hence database changes will be destroyed we only need - // to delete the related repository on the filesystem - if errDelete := util.RemoveAll(repoPath); errDelete != nil { - log.Error("Failed to remove fork repo") - } - } - - needsRollbackInPanic := true - defer func() { - panicErr := recover() - if panicErr == nil { - return - } - - if needsRollbackInPanic { - rollbackFn() - } - panic(panicErr) - }() - - err = db.WithTx(ctx, func(txCtx context.Context) error { - if err = CreateRepositoryByExample(txCtx, doer, owner, repo, false, true); err != nil { + // 1 - Create the repository in the database + err = db.WithTx(ctx, func(ctx context.Context) error { + if err = createRepositoryInDB(ctx, doer, owner, repo, true); err != nil { return err } - - if err = repo_model.IncrementRepoForkNum(txCtx, opts.BaseRepo.ID); err != nil { + if err = repo_model.IncrementRepoForkNum(ctx, opts.BaseRepo.ID); err != nil { return err } // copy lfs files failure should not be ignored - if err = git_model.CopyLFS(txCtx, repo, opts.BaseRepo); err != nil { - return err - } - - needsRollback = true + return git_model.CopyLFS(ctx, repo, opts.BaseRepo) + }) + if err != nil { + return nil, err + } - cloneCmd := git.NewCommand(txCtx, "clone", "--bare") - if opts.SingleBranch != "" { - cloneCmd.AddArguments("--single-branch", "--branch").AddDynamicArguments(opts.SingleBranch) - } - repoPath := repo_model.RepoPath(owner.Name, repo.Name) - if stdout, _, err := cloneCmd.AddDynamicArguments(oldRepoPath, repoPath). - RunStdBytes(&git.RunOpts{Timeout: 10 * time.Minute}); err != nil { - log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, opts.BaseRepo, stdout, err) - return fmt.Errorf("git clone: %w", err) + // last - clean up if something goes wrong + // WARNING: Don't override all later err with local variables + defer func() { + if err != nil { + // we can not use the ctx because it maybe canceled or timeout + cleanupRepository(repo.ID) } + }() - if err := repo_module.CheckDaemonExportOK(txCtx, repo); err != nil { - return fmt.Errorf("checkDaemonExportOK: %w", err) + // 2 - check whether the repository with the same storage exists + var isExist bool + isExist, err = gitrepo.IsRepositoryExist(ctx, repo) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) + return nil, err + } + if isExist { + log.Error("Files already exist in %s and we are not going to adopt or delete.", repo.FullName()) + // Don't return directly, we need err in defer to cleanupRepository + err = repo_model.ErrRepoFilesAlreadyExist{ + Uname: repo.OwnerName, + Name: repo.Name, } + return nil, err + } - if stdout, _, err := git.NewCommand(txCtx, "update-server-info"). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("Fork Repository (git update-server-info) failed for %v:\nStdout: %s\nError: %v", repo, stdout, err) - return fmt.Errorf("git update-server-info: %w", err) - } + // 3 - Clone the repository + cloneCmd := git.NewCommand("clone", "--bare") + if opts.SingleBranch != "" { + cloneCmd.AddArguments("--single-branch", "--branch").AddDynamicArguments(opts.SingleBranch) + } + var stdout []byte + if stdout, _, err = cloneCmd.AddDynamicArguments(opts.BaseRepo.RepoPath(), repo.RepoPath()). + RunStdBytes(ctx, &git.RunOpts{Timeout: 10 * time.Minute}); err != nil { + log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, opts.BaseRepo, stdout, err) + return nil, fmt.Errorf("git clone: %w", err) + } - if err = repo_module.CreateDelegateHooks(repoPath); err != nil { - return fmt.Errorf("createDelegateHooks: %w", err) - } + // 4 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } - gitRepo, err := gitrepo.OpenRepository(txCtx, repo) - if err != nil { - return fmt.Errorf("OpenRepository: %w", err) - } - defer gitRepo.Close() + // 5 - Create hooks + if err = gitrepo.CreateDelegateHooks(ctx, repo); err != nil { + return nil, fmt.Errorf("createDelegateHooks: %w", err) + } - _, err = repo_module.SyncRepoBranchesWithRepo(txCtx, repo, gitRepo, doer.ID) - return err - }) - needsRollbackInPanic = false + // 6 - Sync the repository branches and tags + var gitRepo *git.Repository + gitRepo, err = gitrepo.OpenRepository(ctx, repo) if err != nil { - rollbackFn() - return nil, err + return nil, fmt.Errorf("OpenRepository: %w", err) } + defer gitRepo.Close() + if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doer.ID); err != nil { + return nil, fmt.Errorf("SyncRepoBranchesWithRepo: %w", err) + } + if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { + return nil, fmt.Errorf("Sync releases from git tags failed: %v", err) + } + + // 7 - Update the repository // even if below operations failed, it could be ignored. And they will be retried - if err := repo_module.UpdateRepoSize(ctx, repo); err != nil { + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { log.Error("Failed to update size for repository: %v", err) + err = nil } - if err := repo_model.CopyLanguageStat(ctx, opts.BaseRepo, repo); err != nil { + if err = repo_model.CopyLanguageStat(ctx, opts.BaseRepo, repo); err != nil { log.Error("Copy language stat from oldRepo failed: %v", err) + err = nil } - if err := repo_model.CopyLicense(ctx, opts.BaseRepo, repo); err != nil { + if err = repo_model.CopyLicense(ctx, opts.BaseRepo, repo); err != nil { return nil, err } - gitRepo, err := gitrepo.OpenRepository(ctx, repo) - if err != nil { - log.Error("Open created git repository failed: %v", err) - } else { - defer gitRepo.Close() - if err := repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { - log.Error("Sync releases from git tags failed: %v", err) - } + // 8 - update repository status to be ready + repo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } notify_service.ForkRepository(ctx, doer, opts.BaseRepo, repo) @@ -220,7 +209,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork // ConvertForkToNormalRepository convert the provided repo from a forked repo to normal repo func ConvertForkToNormalRepository(ctx context.Context, repo *repo_model.Repository) error { - err := db.WithTx(ctx, func(ctx context.Context) error { + return db.WithTx(ctx, func(ctx context.Context) error { repo, err := repo_model.GetRepositoryByID(ctx, repo.ID) if err != nil { return err @@ -237,16 +226,8 @@ func ConvertForkToNormalRepository(ctx context.Context, repo *repo_model.Reposit repo.IsFork = false repo.ForkID = 0 - - if err := repo_module.UpdateRepository(ctx, repo, false); err != nil { - log.Error("Unable to update repository %-v whilst converting from fork. Error: %v", repo, err) - return err - } - - return nil + return repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_fork", "fork_id") }) - - return err } type findForksOptions struct { @@ -256,9 +237,11 @@ type findForksOptions struct { } func (opts findForksOptions) ToConds() builder.Cond { - return builder.Eq{"fork_id": opts.RepoID}.And( - repo_model.AccessibleRepositoryCondition(opts.Doer, unit.TypeInvalid), - ) + cond := builder.Eq{"fork_id": opts.RepoID} + if opts.Doer != nil && opts.Doer.IsAdmin { + return cond + } + return cond.And(repo_model.AccessibleRepositoryCondition(opts.Doer, unit.TypeInvalid)) } // FindForks returns all the forks of the repository diff --git a/services/repository/fork_test.go b/services/repository/fork_test.go index 452798b25b..5375f79028 100644 --- a/services/repository/fork_test.go +++ b/services/repository/fork_test.go @@ -4,13 +4,17 @@ package repository import ( + "os" "testing" + "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) @@ -35,7 +39,7 @@ func TestForkRepository(t *testing.T) { assert.False(t, repo_model.IsErrReachLimitOfRepo(err)) // change AllowForkWithoutMaximumLimit to false for the test - setting.Repository.AllowForkWithoutMaximumLimit = false + defer test.MockVariableValue(&setting.Repository.AllowForkWithoutMaximumLimit, false)() // user has reached maximum limit of repositories user.MaxRepoCreation = 0 fork2, err := ForkRepository(git.DefaultContext, user, user, ForkRepoOptions{ @@ -46,3 +50,43 @@ func TestForkRepository(t *testing.T) { assert.Nil(t, fork2) assert.True(t, repo_model.IsErrReachLimitOfRepo(err)) } + +func TestForkRepositoryCleanup(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + // a successful fork + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo10 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}) + + fork, err := ForkRepository(git.DefaultContext, user2, user2, ForkRepoOptions{ + BaseRepo: repo10, + Name: "test", + }) + assert.NoError(t, err) + assert.NotNil(t, fork) + + exist, err := util.IsExist(repo_model.RepoPath(user2.Name, "test")) + assert.NoError(t, err) + assert.True(t, exist) + + err = DeleteRepositoryDirectly(db.DefaultContext, fork.ID) + assert.NoError(t, err) + + // a failed creating because some mock data + // create the repository directory so that the creation will fail after database record created. + assert.NoError(t, os.MkdirAll(repo_model.RepoPath(user2.Name, "test"), os.ModePerm)) + + fork2, err := ForkRepository(db.DefaultContext, user2, user2, ForkRepoOptions{ + BaseRepo: repo10, + Name: "test", + }) + assert.Nil(t, fork2) + assert.Error(t, err) + + // assert the cleanup is successful + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: "test"}) + + exist, err = util.IsExist(repo_model.RepoPath(user2.Name, "test")) + assert.NoError(t, err) + assert.False(t, exist) +} diff --git a/services/repository/generate.go b/services/repository/generate.go index d5c07e9800..867b5d7855 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -17,11 +17,11 @@ import ( git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" @@ -42,10 +42,8 @@ type expansion struct { var defaultTransformers = []transformer{ {Name: "SNAKE", Transform: xstrings.ToSnakeCase}, {Name: "KEBAB", Transform: xstrings.ToKebabCase}, - {Name: "CAMEL", Transform: func(str string) string { - return xstrings.FirstRuneToLower(xstrings.ToCamelCase(str)) - }}, - {Name: "PASCAL", Transform: xstrings.ToCamelCase}, + {Name: "CAMEL", Transform: xstrings.ToCamelCase}, + {Name: "PASCAL", Transform: xstrings.ToPascalCase}, {Name: "LOWER", Transform: strings.ToLower}, {Name: "UPPER", Transform: strings.ToUpper}, {Name: "TITLE", Transform: util.ToTitleCase}, @@ -236,8 +234,8 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r return err } - if stdout, _, err := git.NewCommand(ctx, "remote", "add", "origin").AddDynamicArguments(repo.RepoPath()). - RunStdString(&git.RunOpts{Dir: tmpDir, Env: env}); err != nil { + if stdout, _, err := git.NewCommand("remote", "add", "origin").AddDynamicArguments(repo.RepoPath()). + RunStdString(ctx, &git.RunOpts{Dir: tmpDir, Env: env}); err != nil { log.Error("Unable to add %v as remote origin to temporary repo to %s: stdout %s\nError: %v", repo, tmpDir, stdout, err) return fmt.Errorf("git remote add: %w", err) } @@ -255,48 +253,35 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r return initRepoCommit(ctx, tmpDir, repo, repo.Owner, defaultBranch) } -func generateGitContent(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository) (err error) { - tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name) +// GenerateGitContent generates git content from a template repository +func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) (err error) { + tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-" + generateRepo.Name) if err != nil { - return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.RepoPath(), err) + return fmt.Errorf("failed to create temp dir for repository %s: %w", generateRepo.FullName(), err) } + defer cleanup() - defer func() { - if err := util.RemoveAll(tmpDir); err != nil { - log.Error("RemoveAll: %v", err) - } - }() - - if err = generateRepoCommit(ctx, repo, templateRepo, generateRepo, tmpDir); err != nil { + if err = generateRepoCommit(ctx, generateRepo, templateRepo, generateRepo, tmpDir); err != nil { return fmt.Errorf("generateRepoCommit: %w", err) } // re-fetch repo - if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { + if generateRepo, err = repo_model.GetRepositoryByID(ctx, generateRepo.ID); err != nil { return fmt.Errorf("getRepositoryByID: %w", err) } // if there was no default branch supplied when generating the repo, use the default one from the template - if strings.TrimSpace(repo.DefaultBranch) == "" { - repo.DefaultBranch = templateRepo.DefaultBranch + if strings.TrimSpace(generateRepo.DefaultBranch) == "" { + generateRepo.DefaultBranch = templateRepo.DefaultBranch } - if err = gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + if err = gitrepo.SetDefaultBranch(ctx, generateRepo, generateRepo.DefaultBranch); err != nil { return fmt.Errorf("setDefaultBranch: %w", err) } - if err = UpdateRepository(ctx, repo, false); err != nil { + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, generateRepo, "default_branch"); err != nil { return fmt.Errorf("updateRepository: %w", err) } - return nil -} - -// GenerateGitContent generates git content from a template repository -func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error { - if err := generateGitContent(ctx, generateRepo, templateRepo, generateRepo); err != nil { - return err - } - if err := repo_module.UpdateRepoSize(ctx, generateRepo); err != nil { return fmt.Errorf("failed to update size for repository: %w", err) } @@ -328,58 +313,6 @@ func (gro GenerateRepoOptions) IsValid() bool { gro.IssueLabels || gro.ProtectedBranch // or other items as they are added } -// generateRepository generates a repository from a template -func generateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) { - generateRepo := &repo_model.Repository{ - OwnerID: owner.ID, - Owner: owner, - OwnerName: owner.Name, - Name: opts.Name, - LowerName: strings.ToLower(opts.Name), - Description: opts.Description, - DefaultBranch: opts.DefaultBranch, - IsPrivate: opts.Private, - IsEmpty: !opts.GitContent || templateRepo.IsEmpty, - IsFsckEnabled: templateRepo.IsFsckEnabled, - TemplateID: templateRepo.ID, - TrustModel: templateRepo.TrustModel, - ObjectFormatName: templateRepo.ObjectFormatName, - } - - if err = CreateRepositoryByExample(ctx, doer, owner, generateRepo, false, false); err != nil { - return nil, err - } - - repoPath := generateRepo.RepoPath() - isExist, err := util.IsExist(repoPath) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repoPath, err) - return nil, err - } - if isExist { - return nil, repo_model.ErrRepoFilesAlreadyExist{ - Uname: generateRepo.OwnerName, - Name: generateRepo.Name, - } - } - - if err = repo_module.CheckInitRepository(ctx, owner.Name, generateRepo.Name, generateRepo.ObjectFormatName); err != nil { - return generateRepo, err - } - - if err = repo_module.CheckDaemonExportOK(ctx, generateRepo); err != nil { - return generateRepo, fmt.Errorf("checkDaemonExportOK: %w", err) - } - - if stdout, _, err := git.NewCommand(ctx, "update-server-info"). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("GenerateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", generateRepo, stdout, err) - return generateRepo, fmt.Errorf("error in GenerateRepository(git update-server-info): %w", err) - } - - return generateRepo, nil -} - var fileNameSanitizeRegexp = regexp.MustCompile(`(?i)\.\.|[<>:\"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`) // Sanitize user input to valid OS filenames diff --git a/services/repository/generate_test.go b/services/repository/generate_test.go index b0f97d0ffb..1163c392c9 100644 --- a/services/repository/generate_test.go +++ b/services/repository/generate_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var giteaTemplate = []byte(` @@ -65,3 +66,26 @@ func TestFileNameSanitize(t *testing.T) { assert.Equal(t, "_", fileNameSanitize("\u0000")) assert.Equal(t, "ç›®æ ‡", fileNameSanitize("ç›®æ ‡")) } + +func TestTransformers(t *testing.T) { + cases := []struct { + name string + expected string + }{ + {"SNAKE", "abc_def_xyz"}, + {"KEBAB", "abc-def-xyz"}, + {"CAMEL", "abcDefXyz"}, + {"PASCAL", "AbcDefXyz"}, + {"LOWER", "abc_def-xyz"}, + {"UPPER", "ABC_DEF-XYZ"}, + {"TITLE", "Abc_def-Xyz"}, + } + + input := "Abc_Def-XYZ" + assert.Len(t, defaultTransformers, len(cases)) + for i, c := range cases { + tf := defaultTransformers[i] + require.Equal(t, c.name, tf.Name) + assert.Equal(t, c.expected, tf.Transform(input), "case %s", c.name) + } +} diff --git a/services/repository/gitgraph/graph.go b/services/repository/gitgraph/graph.go new file mode 100644 index 0000000000..d06d18c1b4 --- /dev/null +++ b/services/repository/gitgraph/graph.go @@ -0,0 +1,116 @@ +// Copyright 2016 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgraph + +import ( + "bufio" + "bytes" + "context" + "os" + "strings" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" +) + +// GetCommitGraph return a list of commit (GraphItems) from all branches +func GetCommitGraph(r *git.Repository, page, maxAllowedColors int, hidePRRefs bool, branches, files []string) (*Graph, error) { + format := "DATA:%D|%H|%ad|%h|%s" + + if page == 0 { + page = 1 + } + + graphCmd := git.NewCommand("log", "--graph", "--date-order", "--decorate=full") + + if hidePRRefs { + graphCmd.AddArguments("--exclude=" + git.PullPrefix + "*") + } + + if len(branches) == 0 { + graphCmd.AddArguments("--tags", "--branches") + } + + graphCmd.AddArguments("-C", "-M", "--date=iso-strict"). + AddOptionFormat("-n %d", setting.UI.GraphMaxCommitNum*page). + AddOptionFormat("--pretty=format:%s", format) + + if len(branches) > 0 { + graphCmd.AddDynamicArguments(branches...) + } + if len(files) > 0 { + graphCmd.AddDashesAndList(files...) + } + graph := NewGraph() + + stderr := new(strings.Builder) + stdoutReader, stdoutWriter, err := os.Pipe() + if err != nil { + return nil, err + } + commitsToSkip := setting.UI.GraphMaxCommitNum * (page - 1) + + scanner := bufio.NewScanner(stdoutReader) + + if err := graphCmd.Run(r.Ctx, &git.RunOpts{ + Dir: r.Path, + Stdout: stdoutWriter, + Stderr: stderr, + PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { + _ = stdoutWriter.Close() + defer stdoutReader.Close() + parser := &Parser{} + parser.firstInUse = -1 + parser.maxAllowedColors = maxAllowedColors + if maxAllowedColors > 0 { + parser.availableColors = make([]int, maxAllowedColors) + for i := range parser.availableColors { + parser.availableColors[i] = i + 1 + } + } else { + parser.availableColors = []int{1, 2} + } + for commitsToSkip > 0 && scanner.Scan() { + line := scanner.Bytes() + dataIdx := bytes.Index(line, []byte("DATA:")) + if dataIdx < 0 { + dataIdx = len(line) + } + starIdx := bytes.IndexByte(line, '*') + if starIdx >= 0 && starIdx < dataIdx { + commitsToSkip-- + } + parser.ParseGlyphs(line[:dataIdx]) + } + + row := 0 + + // Skip initial non-commit lines + for scanner.Scan() { + line := scanner.Bytes() + if bytes.IndexByte(line, '*') >= 0 { + if err := parser.AddLineToGraph(graph, row, line); err != nil { + cancel() + return err + } + break + } + parser.ParseGlyphs(line) + } + + for scanner.Scan() { + row++ + line := scanner.Bytes() + if err := parser.AddLineToGraph(graph, row, line); err != nil { + cancel() + return err + } + } + return scanner.Err() + }, + }); err != nil { + return graph, err + } + return graph, nil +} diff --git a/services/repository/gitgraph/graph_models.go b/services/repository/gitgraph/graph_models.go new file mode 100644 index 0000000000..02b0268cd9 --- /dev/null +++ b/services/repository/gitgraph/graph_models.go @@ -0,0 +1,266 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgraph + +import ( + "bytes" + "context" + "fmt" + "strings" + "time" + + asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + asymkey_service "code.gitea.io/gitea/services/asymkey" +) + +// NewGraph creates a basic graph +func NewGraph() *Graph { + graph := &Graph{} + graph.relationCommit = &Commit{ + Row: -1, + Column: -1, + } + graph.Flows = map[int64]*Flow{} + return graph +} + +// Graph represents a collection of flows +type Graph struct { + Flows map[int64]*Flow + Commits []*Commit + MinRow int + MinColumn int + MaxRow int + MaxColumn int + relationCommit *Commit +} + +// Width returns the width of the graph +func (graph *Graph) Width() int { + return graph.MaxColumn - graph.MinColumn + 1 +} + +// Height returns the height of the graph +func (graph *Graph) Height() int { + return graph.MaxRow - graph.MinRow + 1 +} + +// AddGlyph adds glyph to flows +func (graph *Graph) AddGlyph(row, column int, flowID int64, color int, glyph byte) { + flow, ok := graph.Flows[flowID] + if !ok { + flow = NewFlow(flowID, color, row, column) + graph.Flows[flowID] = flow + } + flow.AddGlyph(row, column, glyph) + + if row < graph.MinRow { + graph.MinRow = row + } + if row > graph.MaxRow { + graph.MaxRow = row + } + if column < graph.MinColumn { + graph.MinColumn = column + } + if column > graph.MaxColumn { + graph.MaxColumn = column + } +} + +// AddCommit adds a commit at row, column on flowID with the provided data +func (graph *Graph) AddCommit(row, column int, flowID int64, data []byte) error { + commit, err := NewCommit(row, column, data) + if err != nil { + return err + } + commit.Flow = flowID + graph.Commits = append(graph.Commits, commit) + + graph.Flows[flowID].Commits = append(graph.Flows[flowID].Commits, commit) + return nil +} + +// LoadAndProcessCommits will load the git.Commits for each commit in the graph, +// the associate the commit with the user author, and check the commit verification +// before finally retrieving the latest status +func (graph *Graph) LoadAndProcessCommits(ctx context.Context, repository *repo_model.Repository, gitRepo *git.Repository) error { + var err error + var ok bool + + emails := map[string]*user_model.User{} + keyMap := map[string]bool{} + + for _, c := range graph.Commits { + if len(c.Rev) == 0 { + continue + } + c.Commit, err = gitRepo.GetCommit(c.Rev) + if err != nil { + return fmt.Errorf("GetCommit: %s Error: %w", c.Rev, err) + } + + if c.Commit.Author != nil { + email := c.Commit.Author.Email + if c.User, ok = emails[email]; !ok { + c.User, _ = user_model.GetUserByEmail(ctx, email) + emails[email] = c.User + } + } + + c.Verification = asymkey_service.ParseCommitWithSignature(ctx, c.Commit) + + _ = asymkey_model.CalculateTrustStatus(c.Verification, repository.GetTrustModel(), func(user *user_model.User) (bool, error) { + return repo_model.IsOwnerMemberCollaborator(ctx, repository, user.ID) + }, &keyMap) + + statuses, err := git_model.GetLatestCommitStatus(ctx, repository.ID, c.Commit.ID.String(), db.ListOptionsAll) + if err != nil { + log.Error("GetLatestCommitStatus: %v", err) + } else { + c.Status = git_model.CalcCommitStatus(statuses) + } + } + return nil +} + +// NewFlow creates a new flow +func NewFlow(flowID int64, color, row, column int) *Flow { + return &Flow{ + ID: flowID, + ColorNumber: color, + MinRow: row, + MinColumn: column, + MaxRow: row, + MaxColumn: column, + } +} + +// Flow represents a series of glyphs +type Flow struct { + ID int64 + ColorNumber int + Glyphs []Glyph + Commits []*Commit + MinRow int + MinColumn int + MaxRow int + MaxColumn int +} + +// Color16 wraps the color numbers around mod 16 +func (flow *Flow) Color16() int { + return flow.ColorNumber % 16 +} + +// AddGlyph adds glyph at row and column +func (flow *Flow) AddGlyph(row, column int, glyph byte) { + if row < flow.MinRow { + flow.MinRow = row + } + if row > flow.MaxRow { + flow.MaxRow = row + } + if column < flow.MinColumn { + flow.MinColumn = column + } + if column > flow.MaxColumn { + flow.MaxColumn = column + } + + flow.Glyphs = append(flow.Glyphs, Glyph{ + row, + column, + glyph, + }) +} + +// Glyph represents a co-ordinate and glyph +type Glyph struct { + Row int + Column int + Glyph byte +} + +// RelationCommit represents an empty relation commit +var RelationCommit = &Commit{ + Row: -1, +} + +func parseGitTime(timeStr string) time.Time { + t, err := time.Parse(time.RFC3339, timeStr) + if err != nil { + return time.Unix(0, 0) + } + return t +} + +// NewCommit creates a new commit from a provided line +func NewCommit(row, column int, line []byte) (*Commit, error) { + data := bytes.SplitN(line, []byte("|"), 5) + if len(data) < 5 { + return nil, fmt.Errorf("malformed data section on line %d with commit: %s", row, string(line)) + } + return &Commit{ + Row: row, + Column: column, + // 0 matches git log --pretty=format:%d => ref names, like the --decorate option of git-log(1) + Refs: newRefsFromRefNames(data[0]), + // 1 matches git log --pretty=format:%H => commit hash + Rev: string(data[1]), + // 2 matches git log --pretty=format:%ad => author date (format respects --date= option) + Date: parseGitTime(string(data[2])), + // 3 matches git log --pretty=format:%h => abbreviated commit hash + ShortRev: string(data[3]), + // 4 matches git log --pretty=format:%s => subject + Subject: string(data[4]), + }, nil +} + +func newRefsFromRefNames(refNames []byte) []git.Reference { + refBytes := bytes.Split(refNames, []byte{',', ' '}) + refs := make([]git.Reference, 0, len(refBytes)) + for _, refNameBytes := range refBytes { + if len(refNameBytes) == 0 { + continue + } + refName := string(refNameBytes) + if after, ok := strings.CutPrefix(refName, "tag: "); ok { + refName = after + } else { + refName = strings.TrimPrefix(refName, "HEAD -> ") + } + refs = append(refs, git.Reference{ + Name: refName, + }) + } + return refs +} + +// Commit represents a commit at co-ordinate X, Y with the data +type Commit struct { + Commit *git.Commit + User *user_model.User + Verification *asymkey_model.CommitVerification + Status *git_model.CommitStatus + Flow int64 + Row int + Column int + Refs []git.Reference + Rev string + Date time.Time + ShortRev string + Subject string +} + +// OnlyRelation returns whether this a relation only commit +func (c *Commit) OnlyRelation() bool { + return c.Row == -1 +} diff --git a/services/repository/gitgraph/graph_test.go b/services/repository/gitgraph/graph_test.go new file mode 100644 index 0000000000..93fa1aec6a --- /dev/null +++ b/services/repository/gitgraph/graph_test.go @@ -0,0 +1,695 @@ +// Copyright 2016 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgraph + +import ( + "bytes" + "fmt" + "slices" + "strings" + "testing" + + "code.gitea.io/gitea/modules/git" + + "github.com/stretchr/testify/assert" +) + +func BenchmarkGetCommitGraph(b *testing.B) { + currentRepo, err := git.OpenRepository(git.DefaultContext, ".") + if err != nil || currentRepo == nil { + b.Error("Could not open repository") + } + defer currentRepo.Close() + + for b.Loop() { + graph, err := GetCommitGraph(currentRepo, 1, 0, false, nil, nil) + if err != nil { + b.Error("Could get commit graph") + } + + if len(graph.Commits) < 100 { + b.Error("Should get 100 log lines.") + } + } +} + +func BenchmarkParseCommitString(b *testing.B) { + testString := "* DATA:|4e61bacab44e9b4730e44a6615d04098dd3a8eaf|2016-12-20 21:10:41 +0100|4e61bac|Add route for graph" + + parser := &Parser{} + parser.Reset() + for b.Loop() { + parser.Reset() + graph := NewGraph() + if err := parser.AddLineToGraph(graph, 0, []byte(testString)); err != nil { + b.Error("could not parse teststring") + } + if graph.Flows[1].Commits[0].Rev != "4e61bacab44e9b4730e44a6615d04098dd3a8eaf" { + b.Error("Did not get expected data") + } + } +} + +func BenchmarkParseGlyphs(b *testing.B) { + parser := &Parser{} + parser.Reset() + tgBytes := []byte(testglyphs) + var tg []byte + for b.Loop() { + parser.Reset() + tg = tgBytes + idx := bytes.Index(tg, []byte("\n")) + for idx > 0 { + parser.ParseGlyphs(tg[:idx]) + tg = tg[idx+1:] + idx = bytes.Index(tg, []byte("\n")) + } + } +} + +func TestReleaseUnusedColors(t *testing.T) { + testcases := []struct { + availableColors []int + oldColors []int + firstInUse int // these values have to be either be correct or suggest less is + firstAvailable int // available than possibly is - i.e. you cannot say 10 is available when it + }{ + { + availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + oldColors: []int{1, 1, 1, 1, 1}, + firstAvailable: -1, + firstInUse: 1, + }, + { + availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + oldColors: []int{1, 2, 3, 4}, + firstAvailable: 6, + firstInUse: 0, + }, + { + availableColors: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + oldColors: []int{6, 0, 3, 5, 3, 4, 0, 0}, + firstAvailable: 6, + firstInUse: 0, + }, + { + availableColors: []int{1, 2, 3, 4, 5, 6, 7}, + oldColors: []int{6, 1, 3, 5, 3, 4, 2, 7}, + firstAvailable: -1, + firstInUse: 0, + }, + { + availableColors: []int{1, 2, 3, 4, 5, 6, 7}, + oldColors: []int{6, 0, 3, 5, 3, 4, 2, 7}, + firstAvailable: -1, + firstInUse: 0, + }, + } + for _, testcase := range testcases { + parser := &Parser{} + parser.Reset() + parser.availableColors = append([]int{}, testcase.availableColors...) + parser.oldColors = append(parser.oldColors, testcase.oldColors...) + parser.firstAvailable = testcase.firstAvailable + parser.firstInUse = testcase.firstInUse + parser.releaseUnusedColors() + + if parser.firstAvailable == -1 { + // All in use + for _, color := range parser.availableColors { + found := slices.Contains(parser.oldColors, color) + if !found { + t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should be available but is not", + testcase.availableColors, + testcase.oldColors, + testcase.firstAvailable, + testcase.firstInUse, + parser.availableColors, + parser.oldColors, + parser.firstAvailable, + parser.firstInUse, + color) + } + } + } else if parser.firstInUse != -1 { + // Some in use + for i := parser.firstInUse; i != parser.firstAvailable; i = (i + 1) % len(parser.availableColors) { + color := parser.availableColors[i] + found := slices.Contains(parser.oldColors, color) + if !found { + t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should be available but is not", + testcase.availableColors, + testcase.oldColors, + testcase.firstAvailable, + testcase.firstInUse, + parser.availableColors, + parser.oldColors, + parser.firstAvailable, + parser.firstInUse, + color) + } + } + for i := parser.firstAvailable; i != parser.firstInUse; i = (i + 1) % len(parser.availableColors) { + color := parser.availableColors[i] + found := slices.Contains(parser.oldColors, color) + if found { + t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should not be available but is", + testcase.availableColors, + testcase.oldColors, + testcase.firstAvailable, + testcase.firstInUse, + parser.availableColors, + parser.oldColors, + parser.firstAvailable, + parser.firstInUse, + color) + } + } + } else { + // None in use + for _, color := range parser.oldColors { + if color != 0 { + t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should not be available but is", + testcase.availableColors, + testcase.oldColors, + testcase.firstAvailable, + testcase.firstInUse, + parser.availableColors, + parser.oldColors, + parser.firstAvailable, + parser.firstInUse, + color) + } + } + } + } +} + +func TestParseGlyphs(t *testing.T) { + parser := &Parser{} + parser.Reset() + tgBytes := []byte(testglyphs) + tg := tgBytes + idx := bytes.Index(tg, []byte("\n")) + row := 0 + for idx > 0 { + parser.ParseGlyphs(tg[:idx]) + tg = tg[idx+1:] + idx = bytes.Index(tg, []byte("\n")) + if parser.flows[0] != 1 { + t.Errorf("First column flow should be 1 but was %d", parser.flows[0]) + } + colorToFlow := map[int]int64{} + flowToColor := map[int64]int{} + + for i, flow := range parser.flows { + if flow == 0 { + continue + } + color := parser.colors[i] + + if fColor, in := flowToColor[flow]; in && fColor != color { + t.Errorf("Row %d column %d flow %d has color %d but should be %d", row, i, flow, color, fColor) + } + flowToColor[flow] = color + if cFlow, in := colorToFlow[color]; in && cFlow != flow { + t.Errorf("Row %d column %d flow %d has color %d but conflicts with flow %d", row, i, flow, color, cFlow) + } + colorToFlow[color] = flow + } + row++ + } + assert.Len(t, parser.availableColors, 9) +} + +func TestCommitStringParsing(t *testing.T) { + dataFirstPart := "* DATA:|4e61bacab44e9b4730e44a6615d04098dd3a8eaf|2016-12-20 21:10:41 +0100|4e61bac|" + tests := []struct { + shouldPass bool + testName string + commitMessage string + }{ + {true, "normal", "not a fancy message"}, + {true, "extra pipe", "An extra pipe: |"}, + {true, "extra 'Data:'", "DATA: might be trouble"}, + } + + for _, test := range tests { + t.Run(test.testName, func(t *testing.T) { + testString := fmt.Sprintf("%s%s", dataFirstPart, test.commitMessage) + idx := strings.Index(testString, "DATA:") + commit, err := NewCommit(0, 0, []byte(testString[idx+5:])) + if err != nil && test.shouldPass { + t.Errorf("Could not parse %s", testString) + return + } + + assert.Equal(t, test.commitMessage, commit.Subject) + }) + } +} + +var testglyphs = `* +* +* +* +* +* +* +* +|\ +* | +* | +* | +* | +* | +| * +* | +| * +| |\ +* | | +| | * +| | |\ +* | | \ +|\ \ \ \ +| * | | | +| |\| | | +* | | | | +|/ / / / +| | | * +| * | | +| * | | +| * | | +* | | | +* | | | +* | | | +* | | | +* | | | +|\ \ \ \ +| | * | | +| | |\| | +| | | * | +| | | | * +* | | | | +* | | | | +* | | | | +* | | | | +* | | | | +|\ \ \ \ \ +| * | | | | +|/| | | | | +| | |/ / / +| |/| | | +| | | | * +| * | | | +|/| | | | +| * | | | +|/| | | | +| | |/ / +| |/| | +| * | | +| * | | +| |\ \ \ +| | * | | +| |/| | | +| | | |/ +| | |/| +| * | | +| * | | +| * | | +| | * | +| | |\ \ +| | | * | +| | |/| | +| | | * | +| | | |\ \ +| | | | * | +| | | |/| | +| | * | | | +| | * | | | +| | |\ \ \ \ +| | | * | | | +| | |/| | | | +| | | | | * | +| | | | |/ / +* | | | / / +|/ / / / / +* | | | | +|\ \ \ \ \ +| * | | | | +|/| | | | | +| * | | | | +| * | | | | +| |\ \ \ \ \ +| | | * \ \ \ +| | | |\ \ \ \ +| | | | * | | | +| | | |/| | | | +| | | | | |/ / +| | | | |/| | +* | | | | | | +* | | | | | | +* | | | | | | +| | | | * | | +* | | | | | | +| | * | | | | +| |/| | | | | +* | | | | | | +| |/ / / / / +|/| | | | | +| | | | * | +| | | |/ / +| | |/| | +| * | | | +| | | | * +| | * | | +| | |\ \ \ +| | | * | | +| | |/| | | +| | | |/ / +| | | * | +| | * | | +| | |\ \ \ +| | | * | | +| | |/| | | +| | | |/ / +| | | * | +* | | | | +|\ \ \ \ \ +| * \ \ \ \ +| |\ \ \ \ \ +| | | |/ / / +| | |/| | | +| | | | * | +| | | | * | +* | | | | | +* | | | | | +|/ / / / / +| | | * | +* | | | | +* | | | | +* | | | | +* | | | | +|\ \ \ \ \ +| * | | | | +|/| | | | | +| | * | | | +| | |\ \ \ \ +| | | * | | | +| | |/| | | | +| |/| | |/ / +| | | |/| | +| | | | | * +| |_|_|_|/ +|/| | | | +| | * | | +| |/ / / +* | | | +* | | | +| | * | +* | | | +* | | | +| * | | +| | * | +| * | | +* | | | +|\ \ \ \ +| * | | | +|/| | | | +| |/ / / +| * | | +| |\ \ \ +| | * | | +| |/| | | +| | |/ / +| | * | +| | |\ \ +| | | * | +| | |/| | +* | | | | +* | | | | +|\ \ \ \ \ +| * | | | | +|/| | | | | +| | * | | | +| | * | | | +| | * | | | +| |/ / / / +| * | | | +| |\ \ \ \ +| | * | | | +| |/| | | | +* | | | | | +* | | | | | +* | | | | | +* | | | | | +* | | | | | +| | | | * | +* | | | | | +|\ \ \ \ \ \ +| * | | | | | +|/| | | | | | +| | | | | * | +| | | | |/ / +* | | | | | +|\ \ \ \ \ \ +* | | | | | | +* | | | | | | +| | | | * | | +* | | | | | | +* | | | | | | +|\ \ \ \ \ \ \ +| | |_|_|/ / / +| |/| | | | | +| | | | * | | +| | | | * | | +| | | | * | | +| | | | * | | +| | | | * | | +| | | | * | | +| | | |/ / / +| | | * | | +| | | * | | +| | | * | | +| | |/| | | +| | | * | | +| | |/| | | +| | | |/ / +| | * | | +| |/| | | +| | | * | +| | |/ / +| | * | +| * | | +| |\ \ \ +| * | | | +| | * | | +| |/| | | +| | |/ / +| | * | +| | |\ \ +| | * | | +* | | | | +|\| | | | +| * | | | +| * | | | +| * | | | +| | * | | +| * | | | +| |\| | | +| * | | | +| | * | | +| | * | | +| * | | | +| * | | | +| * | | | +| * | | | +| * | | | +| * | | | +| * | | | +| * | | | +| | * | | +| * | | | +| * | | | +| * | | | +| * | | | +| | * | | +* | | | | +|\| | | | +| | * | | +| * | | | +| |\| | | +| | * | | +| | * | | +| | * | | +| | | * | +* | | | | +|\| | | | +| | * | | +| | |/ / +| * | | +| * | | +| |\| | +* | | | +|\| | | +| | * | +| | * | +| | * | +| * | | +| | * | +| * | | +| | * | +| | * | +| | * | +| * | | +| * | | +| * | | +| * | | +| * | | +| * | | +| * | | +* | | | +|\| | | +| * | | +| |\| | +| | * | +| | |\ \ +* | | | | +|\| | | | +| * | | | +| |\| | | +| | * | | +| | | * | +| | |/ / +* | | | +* | | | +|\| | | +| * | | +| |\| | +| | * | +| | * | +| | * | +| | | * +* | | | +|\| | | +| * | | +| * | | +| | | * +| | | |\ +* | | | | +| |_|_|/ +|/| | | +| * | | +| |\| | +| | * | +| | * | +| | * | +| | * | +| | * | +| * | | +* | | | +|\| | | +| * | | +|/| | | +| |/ / +| * | +| |\ \ +| * | | +| * | | +* | | | +|\| | | +| | * | +| * | | +| * | | +| * | | +* | | | +|\| | | +| * | | +| * | | +| | * | +| | |\ \ +| | |/ / +| |/| | +| * | | +* | | | +|\| | | +| * | | +* | | | +|\| | | +| * | | +| |\ \ \ +| * | | | +| * | | | +| | | * | +| * | | | +| * | | | +| | |/ / +| |/| | +| | * | +* | | | +|\| | | +| * | | +| * | | +| * | | +| * | | +| * | | +| |\ \ \ +* | | | | +|\| | | | +| * | | | +| * | | | +* | | | | +* | | | | +|\| | | | +| | | | * +| | | | |\ +| |_|_|_|/ +|/| | | | +| * | | | +* | | | | +* | | | | +|\| | | | +| * | | | +| |\ \ \ \ +| | | |/ / +| | |/| | +| * | | | +| * | | | +| * | | | +| * | | | +| | * | | +| | | * | +| | |/ / +| |/| | +* | | | +|\| | | +| * | | +| * | | +| * | | +| * | | +| * | | +* | | | +|\| | | +| * | | +| * | | +* | | | +| * | | +| * | | +| * | | +* | | | +* | | | +* | | | +|\| | | +| * | | +* | | | +* | | | +* | | | +* | | | +| | | * +* | | | +|\| | | +| * | | +| * | | +| * | | +` diff --git a/services/repository/gitgraph/parser.go b/services/repository/gitgraph/parser.go new file mode 100644 index 0000000000..f6bf9b0b90 --- /dev/null +++ b/services/repository/gitgraph/parser.go @@ -0,0 +1,336 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package gitgraph + +import ( + "bytes" + "fmt" +) + +// Parser represents a git graph parser. It is stateful containing the previous +// glyphs, detected flows and color assignments. +type Parser struct { + glyphs []byte + oldGlyphs []byte + flows []int64 + oldFlows []int64 + maxFlow int64 + colors []int + oldColors []int + availableColors []int + nextAvailable int + firstInUse int + firstAvailable int + maxAllowedColors int +} + +// Reset resets the internal parser state. +func (parser *Parser) Reset() { + parser.glyphs = parser.glyphs[0:0] + parser.oldGlyphs = parser.oldGlyphs[0:0] + parser.flows = parser.flows[0:0] + parser.oldFlows = parser.oldFlows[0:0] + parser.maxFlow = 0 + parser.colors = parser.colors[0:0] + parser.oldColors = parser.oldColors[0:0] + parser.availableColors = parser.availableColors[0:0] + parser.availableColors = append(parser.availableColors, 1, 2) + parser.nextAvailable = 0 + parser.firstInUse = -1 + parser.firstAvailable = 0 + parser.maxAllowedColors = 0 +} + +// AddLineToGraph adds the line as a row to the graph +func (parser *Parser) AddLineToGraph(graph *Graph, row int, line []byte) error { + idx := bytes.Index(line, []byte("DATA:")) + if idx < 0 { + parser.ParseGlyphs(line) + } else { + parser.ParseGlyphs(line[:idx]) + } + + var err error + commitDone := false + + for column, glyph := range parser.glyphs { + if glyph == ' ' { + continue + } + + flowID := parser.flows[column] + + graph.AddGlyph(row, column, flowID, parser.colors[column], glyph) + + if glyph == '*' { + if commitDone { + if err != nil { + err = fmt.Errorf("double commit on line %d: %s. %w", row, string(line), err) + } else { + err = fmt.Errorf("double commit on line %d: %s", row, string(line)) + } + } + commitDone = true + if idx < 0 { + if err != nil { + err = fmt.Errorf("missing data section on line %d with commit: %s. %w", row, string(line), err) + } else { + err = fmt.Errorf("missing data section on line %d with commit: %s", row, string(line)) + } + continue + } + err2 := graph.AddCommit(row, column, flowID, line[idx+5:]) + if err != nil && err2 != nil { + err = fmt.Errorf("%v %w", err2, err) + continue + } else if err2 != nil { + err = err2 + continue + } + } + } + if !commitDone { + graph.Commits = append(graph.Commits, RelationCommit) + } + return err +} + +func (parser *Parser) releaseUnusedColors() { + if parser.firstInUse > -1 { + // Here we step through the old colors, searching for them in the + // "in-use" section of availableColors (that is, the colors between + // firstInUse and firstAvailable) + // Ensure that the benchmarks are not worsened with proposed changes + stepstaken := 0 + position := parser.firstInUse + for _, color := range parser.oldColors { + if color == 0 { + continue + } + found := false + i := position + for j := stepstaken; i != parser.firstAvailable && j < len(parser.availableColors); j++ { + colorToCheck := parser.availableColors[i] + if colorToCheck == color { + found = true + break + } + i = (i + 1) % len(parser.availableColors) + } + if !found { + // Duplicate color + continue + } + // Swap them around + parser.availableColors[position], parser.availableColors[i] = parser.availableColors[i], parser.availableColors[position] + stepstaken++ + position = (parser.firstInUse + stepstaken) % len(parser.availableColors) + if position == parser.firstAvailable || stepstaken == len(parser.availableColors) { + break + } + } + if stepstaken == len(parser.availableColors) { + parser.firstAvailable = -1 + } else { + parser.firstAvailable = position + if parser.nextAvailable == -1 { + parser.nextAvailable = parser.firstAvailable + } + } + } +} + +// ParseGlyphs parses the provided glyphs and sets the internal state +func (parser *Parser) ParseGlyphs(glyphs []byte) { + // Clean state for parsing this row + parser.glyphs, parser.oldGlyphs = parser.oldGlyphs, parser.glyphs + parser.glyphs = parser.glyphs[0:0] + parser.flows, parser.oldFlows = parser.oldFlows, parser.flows + parser.flows = parser.flows[0:0] + parser.colors, parser.oldColors = parser.oldColors, parser.colors + + // Ensure we have enough flows and colors + parser.colors = parser.colors[0:0] + for range glyphs { + parser.flows = append(parser.flows, 0) + parser.colors = append(parser.colors, 0) + } + + // Copy the provided glyphs in to state.glyphs for safekeeping + parser.glyphs = append(parser.glyphs, glyphs...) + + // release unused colors + parser.releaseUnusedColors() + + for i := len(glyphs) - 1; i >= 0; i-- { + glyph := glyphs[i] + switch glyph { + case '|': + fallthrough + case '*': + parser.setUpFlow(i) + case '/': + parser.setOutFlow(i) + case '\\': + parser.setInFlow(i) + case '_': + parser.setRightFlow(i) + case '.': + fallthrough + case '-': + parser.setLeftFlow(i) + case ' ': + // no-op + default: + parser.newFlow(i) + } + } +} + +func (parser *Parser) takePreviousFlow(i, j int) { + if j < len(parser.oldFlows) && parser.oldFlows[j] > 0 { + parser.flows[i] = parser.oldFlows[j] + parser.oldFlows[j] = 0 + parser.colors[i] = parser.oldColors[j] + parser.oldColors[j] = 0 + } else { + parser.newFlow(i) + } +} + +func (parser *Parser) takeCurrentFlow(i, j int) { + if j < len(parser.flows) && parser.flows[j] > 0 { + parser.flows[i] = parser.flows[j] + parser.colors[i] = parser.colors[j] + } else { + parser.newFlow(i) + } +} + +func (parser *Parser) newFlow(i int) { + parser.maxFlow++ + parser.flows[i] = parser.maxFlow + + // Now give this flow a color + if parser.nextAvailable == -1 { + next := len(parser.availableColors) + if parser.maxAllowedColors < 1 || next < parser.maxAllowedColors { + parser.nextAvailable = next + parser.firstAvailable = next + parser.availableColors = append(parser.availableColors, next+1) + } + } + parser.colors[i] = parser.availableColors[parser.nextAvailable] + if parser.firstInUse == -1 { + parser.firstInUse = parser.nextAvailable + } + parser.availableColors[parser.firstAvailable], parser.availableColors[parser.nextAvailable] = parser.availableColors[parser.nextAvailable], parser.availableColors[parser.firstAvailable] + + parser.nextAvailable = (parser.nextAvailable + 1) % len(parser.availableColors) + parser.firstAvailable = (parser.firstAvailable + 1) % len(parser.availableColors) + + if parser.nextAvailable == parser.firstInUse { + parser.nextAvailable = parser.firstAvailable + } + if parser.nextAvailable == parser.firstInUse { + parser.nextAvailable = -1 + parser.firstAvailable = -1 + } +} + +// setUpFlow handles '|' or '*' +func (parser *Parser) setUpFlow(i int) { + // In preference order: + // + // Previous Row: '\? ' ' |' ' /' + // Current Row: ' | ' ' |' ' | ' + if i > 0 && i-1 < len(parser.oldGlyphs) && parser.oldGlyphs[i-1] == '\\' { + parser.takePreviousFlow(i, i-1) + } else if i < len(parser.oldGlyphs) && (parser.oldGlyphs[i] == '|' || parser.oldGlyphs[i] == '*') { + parser.takePreviousFlow(i, i) + } else if i+1 < len(parser.oldGlyphs) && parser.oldGlyphs[i+1] == '/' { + parser.takePreviousFlow(i, i+1) + } else { + parser.newFlow(i) + } +} + +// setOutFlow handles '/' +func (parser *Parser) setOutFlow(i int) { + // In preference order: + // + // Previous Row: ' |/' ' |_' ' |' ' /' ' _' '\' + // Current Row: '/| ' '/| ' '/ ' '/ ' '/ ' '/' + if i+2 < len(parser.oldGlyphs) && + (parser.oldGlyphs[i+1] == '|' || parser.oldGlyphs[i+1] == '*') && + (parser.oldGlyphs[i+2] == '/' || parser.oldGlyphs[i+2] == '_') && + i+1 < len(parser.glyphs) && + (parser.glyphs[i+1] == '|' || parser.glyphs[i+1] == '*') { + parser.takePreviousFlow(i, i+2) + } else if i+1 < len(parser.oldGlyphs) && + (parser.oldGlyphs[i+1] == '|' || parser.oldGlyphs[i+1] == '*' || + parser.oldGlyphs[i+1] == '/' || parser.oldGlyphs[i+1] == '_') { + parser.takePreviousFlow(i, i+1) + if parser.oldGlyphs[i+1] == '/' { + parser.glyphs[i] = '|' + } + } else if i < len(parser.oldGlyphs) && parser.oldGlyphs[i] == '\\' { + parser.takePreviousFlow(i, i) + } else { + parser.newFlow(i) + } +} + +// setInFlow handles '\' +func (parser *Parser) setInFlow(i int) { + // In preference order: + // + // Previous Row: '| ' '-. ' '| ' '\ ' '/' '---' + // Current Row: '|\' ' \' ' \' ' \' '\' ' \ ' + if i > 0 && i-1 < len(parser.oldGlyphs) && + (parser.oldGlyphs[i-1] == '|' || parser.oldGlyphs[i-1] == '*') && + (parser.glyphs[i-1] == '|' || parser.glyphs[i-1] == '*') { + parser.newFlow(i) + } else if i > 0 && i-1 < len(parser.oldGlyphs) && + (parser.oldGlyphs[i-1] == '|' || parser.oldGlyphs[i-1] == '*' || + parser.oldGlyphs[i-1] == '.' || parser.oldGlyphs[i-1] == '\\') { + parser.takePreviousFlow(i, i-1) + if parser.oldGlyphs[i-1] == '\\' { + parser.glyphs[i] = '|' + } + } else if i < len(parser.oldGlyphs) && parser.oldGlyphs[i] == '/' { + parser.takePreviousFlow(i, i) + } else { + parser.newFlow(i) + } +} + +// setRightFlow handles '_' +func (parser *Parser) setRightFlow(i int) { + // In preference order: + // + // Current Row: '__' '_/' '_|_' '_|/' + if i+1 < len(parser.glyphs) && + (parser.glyphs[i+1] == '_' || parser.glyphs[i+1] == '/') { + parser.takeCurrentFlow(i, i+1) + } else if i+2 < len(parser.glyphs) && + (parser.glyphs[i+1] == '|' || parser.glyphs[i+1] == '*') && + (parser.glyphs[i+2] == '_' || parser.glyphs[i+2] == '/') { + parser.takeCurrentFlow(i, i+2) + } else { + parser.newFlow(i) + } +} + +// setLeftFlow handles '----.' +func (parser *Parser) setLeftFlow(i int) { + if parser.glyphs[i] == '.' { + parser.newFlow(i) + } else if i+1 < len(parser.glyphs) && + (parser.glyphs[i+1] == '-' || parser.glyphs[i+1] == '.') { + parser.takeCurrentFlow(i, i+1) + } else { + parser.newFlow(i) + } +} diff --git a/services/repository/hooks.go b/services/repository/hooks.go index 97e9e290a3..c13b272550 100644 --- a/services/repository/hooks.go +++ b/services/repository/hooks.go @@ -12,7 +12,6 @@ import ( "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" - repo_module "code.gitea.io/gitea/modules/repository" "xorm.io/builder" ) @@ -32,11 +31,11 @@ func SyncRepositoryHooks(ctx context.Context) error { default: } - if err := repo_module.CreateDelegateHooks(repo.RepoPath()); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo); err != nil { return fmt.Errorf("SyncRepositoryHook: %w", err) } if repo.HasWiki() { - if err := repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo.WikiStorageRepo()); err != nil { return fmt.Errorf("SyncRepositoryHook: %w", err) } } diff --git a/services/repository/init.go b/services/repository/init.go index c719e11786..1eeeb4aa4f 100644 --- a/services/repository/init.go +++ b/services/repository/init.go @@ -33,18 +33,21 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi committerName := sig.Name committerEmail := sig.Email - if stdout, _, err := git.NewCommand(ctx, "add", "--all"). - RunStdString(&git.RunOpts{Dir: tmpPath}); err != nil { + if stdout, _, err := git.NewCommand("add", "--all"). + RunStdString(ctx, &git.RunOpts{Dir: tmpPath}); err != nil { log.Error("git add --all failed: Stdout: %s\nError: %v", stdout, err) return fmt.Errorf("git add --all: %w", err) } - cmd := git.NewCommand(ctx, "commit", "--message=Initial commit"). + cmd := git.NewCommand("commit", "--message=Initial commit"). AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email) - sign, keyID, signer, _ := asymkey_service.SignInitialCommit(ctx, tmpPath, u) + sign, key, signer, _ := asymkey_service.SignInitialCommit(ctx, tmpPath, u) if sign { - cmd.AddOptionFormat("-S%s", keyID) + if key.Format != "" { + cmd.AddConfig("gpg.format", key.Format) + } + cmd.AddOptionFormat("-S%s", key.KeyID) if repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { // need to set the committer to the KeyID owner @@ -61,7 +64,7 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi ) if stdout, _, err := cmd. - RunStdString(&git.RunOpts{Dir: tmpPath, Env: env}); err != nil { + RunStdString(ctx, &git.RunOpts{Dir: tmpPath, Env: env}); err != nil { log.Error("Failed to commit: %v: Stdout: %s\nError: %v", cmd.LogString(), stdout, err) return fmt.Errorf("git commit: %w", err) } @@ -70,8 +73,8 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi defaultBranch = setting.Repository.DefaultBranch } - if stdout, _, err := git.NewCommand(ctx, "push", "origin").AddDynamicArguments("HEAD:" + defaultBranch). - RunStdString(&git.RunOpts{Dir: tmpPath, Env: repo_module.InternalPushingEnvironment(u, repo)}); err != nil { + if stdout, _, err := git.NewCommand("push", "origin").AddDynamicArguments("HEAD:"+defaultBranch). + RunStdString(ctx, &git.RunOpts{Dir: tmpPath, Env: repo_module.InternalPushingEnvironment(u, repo)}); err != nil { log.Error("Failed to push back to HEAD: Stdout: %s\nError: %v", stdout, err) return fmt.Errorf("git push: %w", err) } diff --git a/services/repository/lfs_test.go b/services/repository/lfs_test.go index ee0b8f6b89..78ff8c853e 100644 --- a/services/repository/lfs_test.go +++ b/services/repository/lfs_test.go @@ -5,7 +5,6 @@ package repository_test import ( "bytes" - "context" "testing" "time" @@ -36,7 +35,7 @@ func TestGarbageCollectLFSMetaObjects(t *testing.T) { lfsOid := storeObjectInRepo(t, repo.ID, &lfsContent) // gc - err = repo_service.GarbageCollectLFSMetaObjects(context.Background(), repo_service.GarbageCollectLFSMetaObjectsOptions{ + err = repo_service.GarbageCollectLFSMetaObjects(t.Context(), repo_service.GarbageCollectLFSMetaObjectsOptions{ AutoFix: true, OlderThan: time.Now().Add(7 * 24 * time.Hour).Add(5 * 24 * time.Hour), UpdatedLessRecentlyThan: time.Now().Add(7 * 24 * time.Hour).Add(3 * 24 * time.Hour), diff --git a/services/repository/license.go b/services/repository/license.go index 2453be3c87..8622911fa2 100644 --- a/services/repository/license.go +++ b/services/repository/license.go @@ -14,7 +14,6 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/graceful" - "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/queue" @@ -25,7 +24,6 @@ import ( var ( classifier *licenseclassifier.Classifier LicenseFileName = "LICENSE" - licenseAliases map[string]string // licenseUpdaterQueue represents a queue to handle update repo licenses licenseUpdaterQueue *queue.WorkerPoolQueue[*LicenseUpdaterOptions] @@ -38,34 +36,6 @@ func AddRepoToLicenseUpdaterQueue(opts *LicenseUpdaterOptions) error { return licenseUpdaterQueue.Push(opts) } -func loadLicenseAliases() error { - if licenseAliases != nil { - return nil - } - - data, err := options.AssetFS().ReadFile("license", "etc", "license-aliases.json") - if err != nil { - return err - } - err = json.Unmarshal(data, &licenseAliases) - if err != nil { - return err - } - return nil -} - -func ConvertLicenseName(name string) string { - if err := loadLicenseAliases(); err != nil { - return name - } - - v, ok := licenseAliases[name] - if ok { - return v - } - return name -} - func InitLicenseClassifier() error { // threshold should be 0.84~0.86 or the test will be failed classifier = licenseclassifier.NewClassifier(.85) @@ -74,20 +44,13 @@ func InitLicenseClassifier() error { return err } - existLicense := make(container.Set[string]) - if len(licenseFiles) > 0 { - for _, licenseFile := range licenseFiles { - licenseName := ConvertLicenseName(licenseFile) - if existLicense.Contains(licenseName) { - continue - } - existLicense.Add(licenseName) - data, err := options.License(licenseFile) - if err != nil { - return err - } - classifier.AddContent("License", licenseFile, licenseName, data) + for _, licenseFile := range licenseFiles { + licenseName := licenseFile + data, err := options.License(licenseFile) + if err != nil { + return err } + classifier.AddContent("License", licenseName, licenseName, data) } return nil } diff --git a/services/repository/license_test.go b/services/repository/license_test.go index 9d3e0f36e3..eb897f3c03 100644 --- a/services/repository/license_test.go +++ b/services/repository/license_test.go @@ -4,13 +4,13 @@ package repository import ( - "fmt" "strings" "testing" repo_module "code.gitea.io/gitea/modules/repository" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_detectLicense(t *testing.T) { @@ -33,9 +33,7 @@ func Test_detectLicense(t *testing.T) { }, } - repo_module.LoadRepoConfig() - err := loadLicenseAliases() - assert.NoError(t, err) + require.NoError(t, repo_module.LoadRepoConfig()) for _, licenseName := range repo_module.Licenses { license, err := repo_module.GetLicense(licenseName, &repo_module.LicenseValues{ Owner: "Gitea", @@ -46,14 +44,13 @@ func Test_detectLicense(t *testing.T) { assert.NoError(t, err) tests = append(tests, DetectLicenseTest{ - name: fmt.Sprintf("single license test: %s", licenseName), + name: "single license test: " + licenseName, arg: string(license), - want: []string{ConvertLicenseName(licenseName)}, + want: []string{licenseName}, }) } - err = InitLicenseClassifier() - assert.NoError(t, err) + require.NoError(t, InitLicenseClassifier()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { license, err := detectLicense(strings.NewReader(tt.arg)) diff --git a/services/repository/merge_upstream.go b/services/repository/merge_upstream.go index 008e7de385..8d6f11372c 100644 --- a/services/repository/merge_upstream.go +++ b/services/repository/merge_upstream.go @@ -4,7 +4,7 @@ package repository import ( - "context" + "errors" "fmt" issue_model "code.gitea.io/gitea/models/issues" @@ -18,16 +18,24 @@ import ( ) // MergeUpstream merges the base repository's default branch into the fork repository's current branch. -func MergeUpstream(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, branch string) (mergeStyle string, err error) { +func MergeUpstream(ctx reqctx.RequestContext, doer *user_model.User, repo *repo_model.Repository, branch string, ffOnly bool) (mergeStyle string, err error) { if err = repo.MustNotBeArchived(); err != nil { return "", err } if err = repo.GetBaseRepo(ctx); err != nil { return "", err } + divergingInfo, err := GetUpstreamDivergingInfo(ctx, repo, branch) + if err != nil { + return "", err + } + if !divergingInfo.BaseBranchHasNewCommits { + return "up-to-date", nil + } + err = git.Push(ctx, repo.BaseRepo.RepoPath(), git.PushOptions{ Remote: repo.RepoPath(), - Branch: fmt.Sprintf("%s:%s", repo.BaseRepo.DefaultBranch, branch), + Branch: fmt.Sprintf("%s:%s", divergingInfo.BaseBranchName, branch), Env: repo_module.PushingEnvironment(doer, repo), }) if err == nil { @@ -37,6 +45,11 @@ func MergeUpstream(ctx context.Context, doer *user_model.User, repo *repo_model. return "", err } + // If ff_only is requested and fast-forward failed, return error + if ffOnly { + return "", util.NewInvalidArgumentErrorf("fast-forward merge not possible: branch has diverged") + } + // TODO: FakePR: it is somewhat hacky, but it is the only way to "merge" at the moment // ideally in the future the "merge" functions should be refactored to decouple from the PullRequest fakeIssue := &issue_model.Issue{ @@ -59,7 +72,7 @@ func MergeUpstream(ctx context.Context, doer *user_model.User, repo *repo_model. BaseRepoID: repo.BaseRepo.ID, BaseRepo: repo.BaseRepo, HeadBranch: branch, // maybe HeadCommitID is not needed - BaseBranch: repo.BaseRepo.DefaultBranch, + BaseBranch: divergingInfo.BaseBranchName, } fakeIssue.PullRequest = fakePR err = pull.Update(ctx, fakePR, doer, "merge upstream", false) @@ -69,8 +82,15 @@ func MergeUpstream(ctx context.Context, doer *user_model.User, repo *repo_model. return "merge", nil } +// UpstreamDivergingInfo is also used in templates, so it needs to search for all references before changing it. +type UpstreamDivergingInfo struct { + BaseBranchName string + BaseBranchHasNewCommits bool + HeadBranchCommitsBehind int +} + // GetUpstreamDivergingInfo returns the information about the divergence between the fork repository's branch and the base repository's default branch. -func GetUpstreamDivergingInfo(ctx reqctx.RequestContext, forkRepo *repo_model.Repository, forkBranch string) (*BranchDivergingInfo, error) { +func GetUpstreamDivergingInfo(ctx reqctx.RequestContext, forkRepo *repo_model.Repository, forkBranch string) (*UpstreamDivergingInfo, error) { if !forkRepo.IsFork { return nil, util.NewInvalidArgumentErrorf("repo is not a fork") } @@ -83,5 +103,26 @@ func GetUpstreamDivergingInfo(ctx reqctx.RequestContext, forkRepo *repo_model.Re return nil, err } - return GetBranchDivergingInfo(ctx, forkRepo.BaseRepo, forkRepo.BaseRepo.DefaultBranch, forkRepo, forkBranch) + // Do the best to follow the GitHub's behavior, suppose there is a `branch-a` in fork repo: + // * if `branch-a` exists in base repo: try to sync `base:branch-a` to `fork:branch-a` + // * if `branch-a` doesn't exist in base repo: try to sync `base:main` to `fork:branch-a` + info, err := GetBranchDivergingInfo(ctx, forkRepo.BaseRepo, forkBranch, forkRepo, forkBranch) + if err == nil { + return &UpstreamDivergingInfo{ + BaseBranchName: forkBranch, + BaseBranchHasNewCommits: info.BaseHasNewCommits, + HeadBranchCommitsBehind: info.HeadCommitsBehind, + }, nil + } + if errors.Is(err, util.ErrNotExist) { + info, err = GetBranchDivergingInfo(ctx, forkRepo.BaseRepo, forkRepo.BaseRepo.DefaultBranch, forkRepo, forkBranch) + if err == nil { + return &UpstreamDivergingInfo{ + BaseBranchName: forkRepo.BaseRepo.DefaultBranch, + BaseBranchHasNewCommits: info.BaseHasNewCommits, + HeadBranchCommitsBehind: info.HeadCommitsBehind, + }, nil + } + } + return nil, err } diff --git a/services/repository/migrate.go b/services/repository/migrate.go index 6f3a87afa3..0a3dc45339 100644 --- a/services/repository/migrate.go +++ b/services/repository/migrate.go @@ -13,8 +13,10 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/organization" repo_model "code.gitea.io/gitea/models/repo" + unit_model "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migration" @@ -116,14 +118,8 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, repo.Owner = u } - if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { - return repo, fmt.Errorf("checkDaemonExportOK: %w", err) - } - - if stdout, _, err := git.NewCommand(ctx, "update-server-info"). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { - log.Error("MigrateRepositoryGitData(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - return repo, fmt.Errorf("error in MigrateRepositoryGitData(git update-server-info): %w", err) + if err := updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) } gitRepo, err := git.OpenRepository(ctx, repoPath) @@ -140,12 +136,12 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, if !repo.IsEmpty { if len(repo.DefaultBranch) == 0 { // Try to get HEAD branch and set it as default branch. - headBranch, err := gitRepo.GetHEADBranch() + headBranchName, err := git.GetDefaultBranch(ctx, repoPath) if err != nil { return repo, fmt.Errorf("GetHEADBranch: %w", err) } - if headBranch != nil { - repo.DefaultBranch = headBranch.Name + if headBranchName != "" { + repo.DefaultBranch = headBranchName } } @@ -153,9 +149,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err) } + // if releases migration are not requested, we will sync all tags here + // otherwise, the releases sync will be done out of this function if !opts.Releases { - // note: this will greatly improve release (tag) sync - // for pull-mirrors with many tags repo.IsMirror = opts.Mirror if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { log.Error("Failed to synchronize tags to releases for repository: %v", err) @@ -224,15 +220,19 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } repo.IsMirror = true - if err = UpdateRepository(ctx, repo, false); err != nil { + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "num_watches", "is_empty", "default_branch", "default_wiki_branch", "is_mirror"); err != nil { return nil, err } + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + // this is necessary for sync local tags from remote configName := fmt.Sprintf("remote.%s.fetch", mirrorModel.GetRemoteName()) - if stdout, _, err := git.NewCommand(ctx, "config"). + if stdout, _, err := git.NewCommand("config"). AddOptionValues("--add", configName, `+refs/tags/*:refs/tags/*`). - RunStdString(&git.RunOpts{Dir: repoPath}); err != nil { + RunStdString(ctx, &git.RunOpts{Dir: repoPath}); err != nil { log.Error("MigrateRepositoryGitData(git config --add <remote> +refs/tags/*:refs/tags/*) in %v: Stdout: %s\nError: %v", repo, stdout, err) return repo, fmt.Errorf("error in MigrateRepositoryGitData(git config --add <remote> +refs/tags/*:refs/tags/*): %w", err) } @@ -245,15 +245,28 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } } + var enableRepoUnits []repo_model.RepoUnit + if opts.Releases && !unit_model.TypeReleases.UnitGlobalDisabled() { + enableRepoUnits = append(enableRepoUnits, repo_model.RepoUnit{RepoID: repo.ID, Type: unit_model.TypeReleases}) + } + if opts.Wiki && !unit_model.TypeWiki.UnitGlobalDisabled() { + enableRepoUnits = append(enableRepoUnits, repo_model.RepoUnit{RepoID: repo.ID, Type: unit_model.TypeWiki}) + } + if len(enableRepoUnits) > 0 { + err = UpdateRepositoryUnits(ctx, repo, enableRepoUnits, nil) + if err != nil { + return nil, err + } + } return repo, committer.Commit() } // cleanUpMigrateGitConfig removes mirror info which prevents "push --all". // This also removes possible user credentials. func cleanUpMigrateGitConfig(ctx context.Context, repoPath string) error { - cmd := git.NewCommand(ctx, "remote", "rm", "origin") + cmd := git.NewCommand("remote", "rm", "origin") // if the origin does not exist - _, _, err := cmd.RunStdString(&git.RunOpts{ + _, _, err := cmd.RunStdString(ctx, &git.RunOpts{ Dir: repoPath, }) if err != nil && !git.IsRemoteNotExistError(err) { @@ -264,17 +277,16 @@ func cleanUpMigrateGitConfig(ctx context.Context, repoPath string) error { // CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors. func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo_model.Repository, error) { - repoPath := repo.RepoPath() - if err := repo_module.CreateDelegateHooks(repoPath); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo); err != nil { return repo, fmt.Errorf("createDelegateHooks: %w", err) } if repo.HasWiki() { - if err := repo_module.CreateDelegateHooks(repo.WikiPath()); err != nil { + if err := gitrepo.CreateDelegateHooks(ctx, repo.WikiStorageRepo()); err != nil { return repo, fmt.Errorf("createDelegateHooks.(wiki): %w", err) } } - _, _, err := git.NewCommand(ctx, "remote", "rm", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err := git.NewCommand("remote", "rm", "origin").RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) if err != nil && !git.IsRemoteNotExistError(err) { return repo, fmt.Errorf("CleanUpMigrateInfo: %w", err) } diff --git a/services/repository/push.go b/services/repository/push.go index 0ea51f9c07..af3c873d15 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -23,6 +23,7 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" issue_service "code.gitea.io/gitea/services/issue" notify_service "code.gitea.io/gitea/services/notify" pull_service "code.gitea.io/gitea/services/pull" @@ -65,7 +66,7 @@ func PushUpdates(opts []*repo_module.PushUpdateOptions) error { for _, opt := range opts { if opt.IsNewRef() && opt.IsDelRef() { - return fmt.Errorf("Old and new revisions are both NULL") + return errors.New("Old and new revisions are both NULL") } } @@ -133,23 +134,26 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { } else { // is new tag newCommit, err := gitRepo.GetCommit(opts.NewCommitID) if err != nil { - return fmt.Errorf("gitRepo.GetCommit(%s) in %s/%s[%d]: %w", opts.NewCommitID, repo.OwnerName, repo.Name, repo.ID, err) + // in case there is dirty data, for example, the "github.com/git/git" repository has tags pointing to non-existing commits + if !errors.Is(err, util.ErrNotExist) { + log.Error("Unable to get tag commit: gitRepo.GetCommit(%s) in %s/%s[%d]: %v", opts.NewCommitID, repo.OwnerName, repo.Name, repo.ID, err) + } + } else { + commits := repo_module.NewPushCommits() + commits.HeadCommit = repo_module.CommitToPushCommit(newCommit) + commits.CompareURL = repo.ComposeCompareURL(objectFormat.EmptyObjectID().String(), opts.NewCommitID) + + notify_service.PushCommits( + ctx, pusher, repo, + &repo_module.PushUpdateOptions{ + RefFullName: opts.RefFullName, + OldCommitID: objectFormat.EmptyObjectID().String(), + NewCommitID: opts.NewCommitID, + }, commits) + + addTags = append(addTags, tagName) + notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID) } - - commits := repo_module.NewPushCommits() - commits.HeadCommit = repo_module.CommitToPushCommit(newCommit) - commits.CompareURL = repo.ComposeCompareURL(objectFormat.EmptyObjectID().String(), opts.NewCommitID) - - notify_service.PushCommits( - ctx, pusher, repo, - &repo_module.PushUpdateOptions{ - RefFullName: opts.RefFullName, - OldCommitID: objectFormat.EmptyObjectID().String(), - NewCommitID: opts.NewCommitID, - }, commits) - - addTags = append(addTags, tagName) - notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID) } } else if opts.RefFullName.IsBranch() { if pusher == nil || pusher.ID != opts.PusherID { @@ -163,59 +167,25 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { } } - branch := opts.RefFullName.BranchName() if !opts.IsDelRef() { + branch := opts.RefFullName.BranchName() + log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) - go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID) newCommit, err := gitRepo.GetCommit(opts.NewCommitID) if err != nil { return fmt.Errorf("gitRepo.GetCommit(%s) in %s/%s[%d]: %w", opts.NewCommitID, repo.OwnerName, repo.Name, repo.ID, err) } - refName := opts.RefName() - // Push new branch. var l []*git.Commit if opts.IsNewRef() { - if repo.IsEmpty { // Change default branch and empty status only if pushed ref is non-empty branch. - repo.DefaultBranch = refName - repo.IsEmpty = false - if repo.DefaultBranch != setting.Repository.DefaultBranch { - if err := gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { - return err - } - } - // Update the is empty and default_branch columns - if err := repo_model.UpdateRepositoryCols(ctx, repo, "default_branch", "is_empty"); err != nil { - return fmt.Errorf("UpdateRepositoryCols: %w", err) - } - } - - l, err = newCommit.CommitsBeforeLimit(10) - if err != nil { - return fmt.Errorf("newCommit.CommitsBeforeLimit: %w", err) - } - notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID) + l, err = pushNewBranch(ctx, repo, pusher, opts, newCommit) } else { - l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) - if err != nil { - return fmt.Errorf("newCommit.CommitsBeforeUntil: %w", err) - } - - isForcePush, err := newCommit.IsForcePush(opts.OldCommitID) - if err != nil { - log.Error("IsForcePush %s:%s failed: %v", repo.FullName(), branch, err) - } - - if isForcePush { - log.Trace("Push %s is a force push", opts.NewCommitID) - - cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) - } else { - // TODO: increment update the commit count cache but not remove - cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) - } + l, err = pushUpdateBranch(ctx, repo, pusher, opts, newCommit) + } + if err != nil { + return err } // delete cache for divergence @@ -232,36 +202,11 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { commits := repo_module.GitToPushCommits(l) commits.HeadCommit = repo_module.CommitToPushCommit(newCommit) - if err := issue_service.UpdateIssuesCommit(ctx, pusher, repo, commits.Commits, refName); err != nil { + if err := issue_service.UpdateIssuesCommit(ctx, pusher, repo, commits.Commits, opts.RefName()); err != nil { log.Error("updateIssuesCommit: %v", err) } - oldCommitID := opts.OldCommitID - if oldCommitID == objectFormat.EmptyObjectID().String() && len(commits.Commits) > 0 { - oldCommit, err := gitRepo.GetCommit(commits.Commits[len(commits.Commits)-1].Sha1) - if err != nil && !git.IsErrNotExist(err) { - log.Error("unable to GetCommit %s from %-v: %v", oldCommitID, repo, err) - } - if oldCommit != nil { - for i := 0; i < oldCommit.ParentCount(); i++ { - commitID, _ := oldCommit.ParentID(i) - if !commitID.IsZero() { - oldCommitID = commitID.String() - break - } - } - } - } - - if oldCommitID == objectFormat.EmptyObjectID().String() && repo.DefaultBranch != branch { - oldCommitID = repo.DefaultBranch - } - - if oldCommitID != objectFormat.EmptyObjectID().String() { - commits.CompareURL = repo.ComposeCompareURL(oldCommitID, opts.NewCommitID) - } else { - commits.CompareURL = "" - } + commits.CompareURL = getCompareURL(repo, gitRepo, objectFormat, commits.Commits, opts) if len(commits.Commits) > setting.UI.FeedMaxCommitNum { commits.Commits = commits.Commits[:setting.UI.FeedMaxCommitNum] @@ -274,12 +219,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { log.Error("repo_module.CacheRef %s/%s failed: %v", repo.ID, branch, err) } } else { - notify_service.DeleteRef(ctx, pusher, repo, opts.RefFullName) - - if err := pull_service.AdjustPullsCausedByBranchDeleted(ctx, pusher, repo, branch); err != nil { - // close all related pulls - log.Error("close related pull request failed: %v", err) - } + pushDeleteBranch(ctx, repo, pusher, opts) } // Even if user delete a branch on a repository which he didn't watch, he will be watch that. @@ -290,8 +230,11 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { log.Trace("Non-tag and non-branch commits pushed.") } } - if err := PushUpdateAddDeleteTags(ctx, repo, gitRepo, addTags, delTags); err != nil { - return fmt.Errorf("PushUpdateAddDeleteTags: %w", err) + + if len(addTags)+len(delTags) > 0 { + if err := PushUpdateAddDeleteTags(ctx, repo, gitRepo, pusher, addTags, delTags); err != nil { + return fmt.Errorf("PushUpdateAddDeleteTags: %w", err) + } } // Change repository last updated time. @@ -302,18 +245,114 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { return nil } +func getCompareURL(repo *repo_model.Repository, gitRepo *git.Repository, objectFormat git.ObjectFormat, commits []*repo_module.PushCommit, opts *repo_module.PushUpdateOptions) string { + oldCommitID := opts.OldCommitID + if oldCommitID == objectFormat.EmptyObjectID().String() && len(commits) > 0 { + oldCommit, err := gitRepo.GetCommit(commits[len(commits)-1].Sha1) + if err != nil && !git.IsErrNotExist(err) { + log.Error("unable to GetCommit %s from %-v: %v", oldCommitID, repo, err) + } + if oldCommit != nil { + for i := 0; i < oldCommit.ParentCount(); i++ { + commitID, _ := oldCommit.ParentID(i) + if !commitID.IsZero() { + oldCommitID = commitID.String() + break + } + } + } + } + + if oldCommitID == objectFormat.EmptyObjectID().String() && repo.DefaultBranch != opts.RefFullName.BranchName() { + oldCommitID = repo.DefaultBranch + } + + if oldCommitID != objectFormat.EmptyObjectID().String() { + return repo.ComposeCompareURL(oldCommitID, opts.NewCommitID) + } + return "" +} + +func pushNewBranch(ctx context.Context, repo *repo_model.Repository, pusher *user_model.User, opts *repo_module.PushUpdateOptions, newCommit *git.Commit) ([]*git.Commit, error) { + if repo.IsEmpty { // Change default branch and empty status only if pushed ref is non-empty branch. + repo.DefaultBranch = opts.RefName() + repo.IsEmpty = false + if repo.DefaultBranch != setting.Repository.DefaultBranch { + if err := gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + return nil, err + } + } + // Update the is empty and default_branch columns + if err := repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "default_branch", "is_empty"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) + } + } + + l, err := newCommit.CommitsBeforeLimit(10) + if err != nil { + return nil, fmt.Errorf("newCommit.CommitsBeforeLimit: %w", err) + } + notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID) + return l, nil +} + +func pushUpdateBranch(_ context.Context, repo *repo_model.Repository, pusher *user_model.User, opts *repo_module.PushUpdateOptions, newCommit *git.Commit) ([]*git.Commit, error) { + l, err := newCommit.CommitsBeforeUntil(opts.OldCommitID) + if err != nil { + return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %w", err) + } + + branch := opts.RefFullName.BranchName() + + isForcePush, err := newCommit.IsForcePush(opts.OldCommitID) + if err != nil { + log.Error("IsForcePush %s:%s failed: %v", repo.FullName(), branch, err) + } + + // only update branch can trigger pull request task because the pull request hasn't been created yet when creating a branch + go pull_service.AddTestPullRequestTask(pull_service.TestPullRequestOptions{ + RepoID: repo.ID, + Doer: pusher, + Branch: branch, + IsSync: true, + IsForcePush: isForcePush, + OldCommitID: opts.OldCommitID, + NewCommitID: opts.NewCommitID, + }) + + if isForcePush { + log.Trace("Push %s is a force push", opts.NewCommitID) + + cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) + } else { + // TODO: increment update the commit count cache but not remove + cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true)) + } + + return l, nil +} + +func pushDeleteBranch(ctx context.Context, repo *repo_model.Repository, pusher *user_model.User, opts *repo_module.PushUpdateOptions) { + notify_service.DeleteRef(ctx, pusher, repo, opts.RefFullName) + + if err := pull_service.AdjustPullsCausedByBranchDeleted(ctx, pusher, repo, opts.RefFullName.BranchName()); err != nil { + // close all related pulls + log.Error("close related pull request failed: %v", err) + } +} + // PushUpdateAddDeleteTags updates a number of added and delete tags -func PushUpdateAddDeleteTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, addTags, delTags []string) error { +func PushUpdateAddDeleteTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, pusher *user_model.User, addTags, delTags []string) error { return db.WithTx(ctx, func(ctx context.Context) error { - if err := repo_model.PushUpdateDeleteTagsContext(ctx, repo, delTags); err != nil { + if err := repo_model.PushUpdateDeleteTags(ctx, repo, delTags); err != nil { return err } - return pushUpdateAddTags(ctx, repo, gitRepo, addTags) + return pushUpdateAddTags(ctx, repo, gitRepo, pusher, addTags) }) } // pushUpdateAddTags updates a number of add tags -func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tags []string) error { +func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, pusher *user_model.User, tags []string) error { if len(tags) == 0 { return nil } @@ -339,14 +378,12 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo newReleases := make([]*repo_model.Release, 0, len(lowerTags)-len(relMap)) - emailToUser := make(map[string]*user_model.User) - for i, lowerTag := range lowerTags { tag, err := gitRepo.GetTag(tags[i]) if err != nil { return fmt.Errorf("GetTag: %w", err) } - commit, err := tag.Commit(gitRepo) + commit, err := gitRepo.GetTagCommit(tag.Name) if err != nil { return fmt.Errorf("Commit: %w", err) } @@ -358,29 +395,12 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo if sig == nil { sig = commit.Committer } - var author *user_model.User - createdAt := time.Unix(1, 0) + createdAt := time.Unix(1, 0) if sig != nil { - var ok bool - author, ok = emailToUser[sig.Email] - if !ok { - author, err = user_model.GetUserByEmail(ctx, sig.Email) - if err != nil && !user_model.IsErrUserNotExist(err) { - return fmt.Errorf("GetUserByEmail: %w", err) - } - if author != nil { - emailToUser[sig.Email] = author - } - } createdAt = sig.When } - commitsCount, err := commit.CommitsCount() - if err != nil { - return fmt.Errorf("CommitsCount: %w", err) - } - rel, has := relMap[lowerTag] parts := strings.SplitN(tag.Message, "\n", 2) @@ -396,31 +416,26 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo LowerTagName: lowerTag, Target: "", Sha1: commit.ID.String(), - NumCommits: commitsCount, + NumCommits: -1, // the commits count will be updated when the UI needs it Note: note, IsDraft: false, IsPrerelease: false, IsTag: true, + PublisherID: pusher.ID, CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), } - if author != nil { - rel.PublisherID = author.ID - } newReleases = append(newReleases, rel) } else { rel.Sha1 = commit.ID.String() rel.CreatedUnix = timeutil.TimeStamp(createdAt.Unix()) - rel.NumCommits = commitsCount if rel.IsTag { rel.Title = parts[0] rel.Note = note - if author != nil { - rel.PublisherID = author.ID - } } else { rel.IsDraft = false } + rel.PublisherID = pusher.ID if err = repo_model.UpdateRelease(ctx, rel); err != nil { return fmt.Errorf("Update: %w", err) } diff --git a/services/repository/repository.go b/services/repository/repository.go index 59b4491132..e574dc6c01 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -5,23 +5,29 @@ package repository import ( "context" + "errors" "fmt" + "os" + "path/filepath" + "strings" + activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/organization" - packages_model "code.gitea.io/gitea/models/packages" + access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" - system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/graceful" + issue_indexer "code.gitea.io/gitea/modules/indexer/issues" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/queue" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" pull_service "code.gitea.io/gitea/services/pull" ) @@ -41,7 +47,7 @@ type WebSearchResults struct { // CreateRepository creates a repository for the user/organization. func CreateRepository(ctx context.Context, doer, owner *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { - repo, err := CreateRepositoryDirectly(ctx, doer, owner, opts) + repo, err := CreateRepositoryDirectly(ctx, doer, owner, opts, true) if err != nil { // No need to rollback here we should do this in CreateRepository... return nil, err @@ -63,11 +69,7 @@ func DeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_mod notify_service.DeleteRepository(ctx, doer, repo) } - if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { - return err - } - - return packages_model.UnlinkRepositoryFromAllPackages(ctx, repo.ID) + return DeleteRepositoryDirectly(ctx, repo.ID) } // PushCreateRepo creates a repository when a new repository is pushed to an appropriate namespace @@ -77,10 +79,10 @@ func PushCreateRepo(ctx context.Context, authUser, owner *user_model.User, repoN if ok, err := organization.CanCreateOrgRepo(ctx, owner.ID, authUser.ID); err != nil { return nil, err } else if !ok { - return nil, fmt.Errorf("cannot push-create repository for org") + return nil, errors.New("cannot push-create repository for org") } } else if authUser.ID != owner.ID { - return nil, fmt.Errorf("cannot push-create repository for another user") + return nil, errors.New("cannot push-create repository for another user") } } @@ -99,15 +101,13 @@ func PushCreateRepo(ctx context.Context, authUser, owner *user_model.User, repoN func Init(ctx context.Context) error { licenseUpdaterQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_license_updater", repoLicenseUpdater) if licenseUpdaterQueue == nil { - return fmt.Errorf("unable to create repo_license_updater queue") + return errors.New("unable to create repo_license_updater queue") } go graceful.GetManager().RunWithCancel(licenseUpdaterQueue) if err := repo_module.LoadRepoConfig(); err != nil { return err } - system_model.RemoveAllWithNotice(ctx, "Clean up temporary repository uploads", setting.Repository.Upload.TempPath) - system_model.RemoveAllWithNotice(ctx, "Clean up temporary repositories", repo_module.LocalCopyPath()) if err := initPushQueue(); err != nil { return err } @@ -116,42 +116,107 @@ func Init(ctx context.Context) error { // UpdateRepository updates a repository func UpdateRepository(ctx context.Context, repo *repo_model.Repository, visibilityChanged bool) (err error) { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - if err = repo_module.UpdateRepository(ctx, repo, visibilityChanged); err != nil { - return fmt.Errorf("updateRepository: %w", err) - } - - return committer.Commit() + return db.WithTx(ctx, func(ctx context.Context) error { + if err = updateRepository(ctx, repo, visibilityChanged); err != nil { + return fmt.Errorf("updateRepository: %w", err) + } + return nil + }) } -func UpdateRepositoryVisibility(ctx context.Context, repo *repo_model.Repository, isPrivate bool) (err error) { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } +func MakeRepoPublic(ctx context.Context, repo *repo_model.Repository) (err error) { + return db.WithTx(ctx, func(ctx context.Context) error { + repo.IsPrivate = false + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_private"); err != nil { + return err + } - defer committer.Close() + if err = repo.LoadOwner(ctx); err != nil { + return fmt.Errorf("LoadOwner: %w", err) + } + if repo.Owner.IsOrganization() { + // Organization repository need to recalculate access table when visibility is changed. + if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { + return fmt.Errorf("recalculateTeamAccesses: %w", err) + } + } - repo.IsPrivate = isPrivate + // Create/Remove git-daemon-export-ok for git-daemon... + if err := checkDaemonExportOK(ctx, repo); err != nil { + return err + } - if err = repo_module.UpdateRepository(ctx, repo, true); err != nil { - return fmt.Errorf("UpdateRepositoryVisibility: %w", err) - } + forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) + if err != nil { + return fmt.Errorf("getRepositoriesByForkID: %w", err) + } - return committer.Commit() -} + if repo.Owner.Visibility != structs.VisibleTypePrivate { + for i := range forkRepos { + if err = MakeRepoPublic(ctx, forkRepos[i]); err != nil { + return fmt.Errorf("MakeRepoPublic[%d]: %w", forkRepos[i].ID, err) + } + } + } -func MakeRepoPublic(ctx context.Context, repo *repo_model.Repository) (err error) { - return UpdateRepositoryVisibility(ctx, repo, false) + // If visibility is changed, we need to update the issue indexer. + // Since the data in the issue indexer have field to indicate if the repo is public or not. + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + + return nil + }) } func MakeRepoPrivate(ctx context.Context, repo *repo_model.Repository) (err error) { - return UpdateRepositoryVisibility(ctx, repo, true) + return db.WithTx(ctx, func(ctx context.Context) error { + repo.IsPrivate = true + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_private"); err != nil { + return err + } + + if err = repo.LoadOwner(ctx); err != nil { + return fmt.Errorf("LoadOwner: %w", err) + } + if repo.Owner.IsOrganization() { + // Organization repository need to recalculate access table when visibility is changed. + if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { + return fmt.Errorf("recalculateTeamAccesses: %w", err) + } + } + + // If repo has become private, we need to set its actions to private. + _, err = db.GetEngine(ctx).Where("repo_id = ?", repo.ID).Cols("is_private").Update(&activities_model.Action{ + IsPrivate: true, + }) + if err != nil { + return err + } + + if err = repo_model.ClearRepoStars(ctx, repo.ID); err != nil { + return err + } + + // Create/Remove git-daemon-export-ok for git-daemon... + if err := checkDaemonExportOK(ctx, repo); err != nil { + return err + } + + forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) + if err != nil { + return fmt.Errorf("getRepositoriesByForkID: %w", err) + } + for i := range forkRepos { + if err = MakeRepoPrivate(ctx, forkRepos[i]); err != nil { + return fmt.Errorf("MakeRepoPrivate[%d]: %w", forkRepos[i].ID, err) + } + } + + // If visibility is changed, we need to update the issue indexer. + // Since the data in the issue indexer have field to indicate if the repo is public or not. + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + + return nil + }) } // LinkedRepository returns the linked repo if any @@ -177,3 +242,97 @@ func LinkedRepository(ctx context.Context, a *repo_model.Attachment) (*repo_mode } return nil, -1, nil } + +// checkDaemonExportOK creates/removes git-daemon-export-ok for git-daemon... +func checkDaemonExportOK(ctx context.Context, repo *repo_model.Repository) error { + if err := repo.LoadOwner(ctx); err != nil { + return err + } + + // Create/Remove git-daemon-export-ok for git-daemon... + daemonExportFile := filepath.Join(repo.RepoPath(), `git-daemon-export-ok`) + + isExist, err := util.IsExist(daemonExportFile) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err) + return err + } + + isPublic := !repo.IsPrivate && repo.Owner.Visibility == structs.VisibleTypePublic + if !isPublic && isExist { + if err = util.Remove(daemonExportFile); err != nil { + log.Error("Failed to remove %s: %v", daemonExportFile, err) + } + } else if isPublic && !isExist { + if f, err := os.Create(daemonExportFile); err != nil { + log.Error("Failed to create %s: %v", daemonExportFile, err) + } else { + f.Close() + } + } + + return nil +} + +// updateRepository updates a repository with db context +func updateRepository(ctx context.Context, repo *repo_model.Repository, visibilityChanged bool) (err error) { + repo.LowerName = strings.ToLower(repo.Name) + + e := db.GetEngine(ctx) + + if _, err = e.ID(repo.ID).NoAutoTime().AllCols().Update(repo); err != nil { + return fmt.Errorf("update: %w", err) + } + + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + + if visibilityChanged { + if err = repo.LoadOwner(ctx); err != nil { + return fmt.Errorf("LoadOwner: %w", err) + } + if repo.Owner.IsOrganization() { + // Organization repository need to recalculate access table when visibility is changed. + if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { + return fmt.Errorf("recalculateTeamAccesses: %w", err) + } + } + + // If repo has become private, we need to set its actions to private. + if repo.IsPrivate { + _, err = e.Where("repo_id = ?", repo.ID).Cols("is_private").Update(&activities_model.Action{ + IsPrivate: true, + }) + if err != nil { + return err + } + + if err = repo_model.ClearRepoStars(ctx, repo.ID); err != nil { + return err + } + } + + // Create/Remove git-daemon-export-ok for git-daemon... + if err := checkDaemonExportOK(ctx, repo); err != nil { + return err + } + + forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) + if err != nil { + return fmt.Errorf("getRepositoriesByForkID: %w", err) + } + for i := range forkRepos { + forkRepos[i].IsPrivate = repo.IsPrivate || repo.Owner.Visibility == structs.VisibleTypePrivate + if err = updateRepository(ctx, forkRepos[i], true); err != nil { + return fmt.Errorf("updateRepository[%d]: %w", forkRepos[i].ID, err) + } + } + + // If visibility is changed, we need to update the issue indexer. + // Since the data in the issue indexer have field to indicate if the repo is public or not. + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + } + + return nil +} diff --git a/services/repository/repository_test.go b/services/repository/repository_test.go index 892a11a23e..8f9fdf8fa1 100644 --- a/services/repository/repository_test.go +++ b/services/repository/repository_test.go @@ -6,6 +6,7 @@ package repository import ( "testing" + activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" @@ -40,3 +41,23 @@ func TestLinkedRepository(t *testing.T) { }) } } + +func TestUpdateRepositoryVisibilityChanged(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + // Get sample repo and change visibility + repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 9) + assert.NoError(t, err) + repo.IsPrivate = true + + // Update it + err = updateRepository(db.DefaultContext, repo, true) + assert.NoError(t, err) + + // Check visibility of action has become private + act := activities_model.Action{} + _, err = db.GetEngine(db.DefaultContext).ID(3).Get(&act) + + assert.NoError(t, err) + assert.True(t, act.IsPrivate) +} diff --git a/services/repository/setting.go b/services/repository/setting.go index b82f24271e..e0c787dd2d 100644 --- a/services/repository/setting.go +++ b/services/repository/setting.go @@ -7,7 +7,6 @@ import ( "context" "slices" - actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" @@ -29,7 +28,7 @@ func UpdateRepositoryUnits(ctx context.Context, repo *repo_model.Repository, uni } if slices.Contains(deleteUnitTypes, unit.TypeActions) { - if err := actions_model.CleanRepoScheduleTasks(ctx, repo); err != nil { + if err := actions_service.CleanRepoScheduleTasks(ctx, repo); err != nil { log.Error("CleanRepoScheduleTasks: %v", err) } } diff --git a/services/repository/template.go b/services/repository/template.go index 36a680c8e2..6906a60083 100644 --- a/services/repository/template.go +++ b/services/repository/template.go @@ -5,12 +5,17 @@ package repository import ( "context" + "fmt" + "strings" "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" notify_service "code.gitea.io/gitea/services/notify" ) @@ -63,70 +68,124 @@ func GenerateProtectedBranch(ctx context.Context, templateRepo, generateRepo *re // GenerateRepository generates a repository from a template func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) { - if !doer.IsAdmin && !owner.CanCreateRepo() { + if !doer.CanCreateRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ Limit: owner.MaxRepoCreation, } } - var generateRepo *repo_model.Repository - if err = db.WithTx(ctx, func(ctx context.Context) error { - generateRepo, err = generateRepository(ctx, doer, owner, templateRepo, opts) + generateRepo := &repo_model.Repository{ + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, + Name: opts.Name, + LowerName: strings.ToLower(opts.Name), + Description: opts.Description, + DefaultBranch: opts.DefaultBranch, + IsPrivate: opts.Private, + IsEmpty: !opts.GitContent || templateRepo.IsEmpty, + IsFsckEnabled: templateRepo.IsFsckEnabled, + TemplateID: templateRepo.ID, + TrustModel: templateRepo.TrustModel, + ObjectFormatName: templateRepo.ObjectFormatName, + Status: repo_model.RepositoryBeingMigrated, + } + + // 1 - Create the repository in the database + if err := db.WithTx(ctx, func(ctx context.Context) error { + return createRepositoryInDB(ctx, doer, owner, generateRepo, false) + }); err != nil { + return nil, err + } + + // last - clean up the repository if something goes wrong + defer func() { if err != nil { - return err + // we can not use the ctx because it maybe canceled or timeout + cleanupRepository(generateRepo.ID) } + }() - // Git Content - if opts.GitContent && !templateRepo.IsEmpty { - if err = GenerateGitContent(ctx, templateRepo, generateRepo); err != nil { - return err - } + // 2 - check whether the repository with the same storage exists + isExist, err := gitrepo.IsRepositoryExist(ctx, generateRepo) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", generateRepo.FullName(), err) + return nil, err + } + if isExist { + // Don't return directly, we need err in defer to cleanupRepository + err = repo_model.ErrRepoFilesAlreadyExist{ + Uname: generateRepo.OwnerName, + Name: generateRepo.Name, } + return nil, err + } - // Topics - if opts.Topics { - if err = repo_model.GenerateTopics(ctx, templateRepo, generateRepo); err != nil { - return err - } + // 3 -Init git bare new repository. + if err = git.InitRepository(ctx, generateRepo.RepoPath(), true, generateRepo.ObjectFormatName); err != nil { + return nil, fmt.Errorf("git.InitRepository: %w", err) + } else if err = gitrepo.CreateDelegateHooks(ctx, generateRepo); err != nil { + return nil, fmt.Errorf("createDelegateHooks: %w", err) + } + + // 4 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, generateRepo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } + + // 5 - generate the repository contents according to the template + // Git Content + if opts.GitContent && !templateRepo.IsEmpty { + if err = GenerateGitContent(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Git Hooks - if opts.GitHooks { - if err = GenerateGitHooks(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Topics + if opts.Topics { + if err = repo_model.GenerateTopics(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Webhooks - if opts.Webhooks { - if err = GenerateWebhooks(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Git Hooks + if opts.GitHooks { + if err = GenerateGitHooks(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Avatar - if opts.Avatar && len(templateRepo.Avatar) > 0 { - if err = generateAvatar(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Webhooks + if opts.Webhooks { + if err = GenerateWebhooks(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Issue Labels - if opts.IssueLabels { - if err = GenerateIssueLabels(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Avatar + if opts.Avatar && len(templateRepo.Avatar) > 0 { + if err = generateAvatar(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - if opts.ProtectedBranch { - if err = GenerateProtectedBranch(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Issue Labels + if opts.IssueLabels { + if err = GenerateIssueLabels(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - return nil - }); err != nil { - return nil, err + if opts.ProtectedBranch { + if err = GenerateProtectedBranch(ctx, templateRepo, generateRepo); err != nil { + return nil, err + } + } + + // 6 - update repository status to be ready + generateRepo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, generateRepo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } notify_service.CreateRepository(ctx, doer, owner, generateRepo) diff --git a/services/repository/transfer.go b/services/repository/transfer.go index 9ef28ddeb9..5ad63cca67 100644 --- a/services/repository/transfer.go +++ b/services/repository/transfer.go @@ -17,29 +17,31 @@ import ( project_model "code.gitea.io/gitea/models/project" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/globallock" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" ) -func getRepoWorkingLockKey(repoID int64) string { - return fmt.Sprintf("repo_working_%d", repoID) +type LimitReachedError struct{ Limit int } + +func (LimitReachedError) Error() string { + return "Repository limit has been reached" } -// TransferOwnership transfers all corresponding setting from old user to new one. -func TransferOwnership(ctx context.Context, doer, newOwner *user_model.User, repo *repo_model.Repository, teams []*organization.Team) error { - if err := repo.LoadOwner(ctx); err != nil { - return err - } - for _, team := range teams { - if newOwner.ID != team.OrgID { - return fmt.Errorf("team %d does not belong to organization", team.ID) - } - } +func IsRepositoryLimitReached(err error) bool { + _, ok := err.(LimitReachedError) + return ok +} - oldOwner := repo.Owner +func getRepoWorkingLockKey(repoID int64) string { + return fmt.Sprintf("repo_working_%d", repoID) +} +// AcceptTransferOwnership transfers all corresponding setting from old user to new one. +func AcceptTransferOwnership(ctx context.Context, repo *repo_model.Repository, doer *user_model.User) error { releaser, err := globallock.Lock(ctx, getRepoWorkingLockKey(repo.ID)) if err != nil { log.Error("lock.Lock(): %v", err) @@ -47,29 +49,49 @@ func TransferOwnership(ctx context.Context, doer, newOwner *user_model.User, rep } defer releaser() - if err := transferOwnership(ctx, doer, newOwner.Name, repo); err != nil { - return err - } - releaser() - - newRepo, err := repo_model.GetRepositoryByID(ctx, repo.ID) + repoTransfer, err := repo_model.GetPendingRepositoryTransfer(ctx, repo) if err != nil { return err } - for _, team := range teams { - if err := addRepositoryToTeam(ctx, team, newRepo); err != nil { + oldOwnerName := repo.OwnerName + + if err := db.WithTx(ctx, func(ctx context.Context) error { + if err := repoTransfer.LoadAttributes(ctx); err != nil { return err } + + if !doer.CanCreateRepoIn(repoTransfer.Recipient) { + limit := util.Iif(repoTransfer.Recipient.MaxRepoCreation >= 0, repoTransfer.Recipient.MaxRepoCreation, setting.Repository.MaxCreationLimit) + return LimitReachedError{Limit: limit} + } + + if !repoTransfer.CanUserAcceptOrRejectTransfer(ctx, doer) { + return util.ErrPermissionDenied + } + + if err := repo.LoadOwner(ctx); err != nil { + return err + } + for _, team := range repoTransfer.Teams { + if repoTransfer.Recipient.ID != team.OrgID { + return fmt.Errorf("team %d does not belong to organization", team.ID) + } + } + + return transferOwnership(ctx, repoTransfer.Doer, repoTransfer.Recipient.Name, repo, repoTransfer.Teams) + }); err != nil { + return err } + releaser() - notify_service.TransferRepository(ctx, doer, repo, oldOwner.Name) + notify_service.TransferRepository(ctx, doer, repo, oldOwnerName) return nil } // transferOwnership transfers all corresponding repository items from old user to new one. -func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName string, repo *repo_model.Repository) (err error) { +func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName string, repo *repo_model.Repository, teams []*organization.Team) (err error) { repoRenamed := false wikiRenamed := false oldOwnerName := doer.Name @@ -138,7 +160,7 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName repo.OwnerName = newOwner.Name // Update repository. - if _, err := sess.ID(repo.ID).Update(repo); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "owner_id", "owner_name"); err != nil { return fmt.Errorf("update owner: %w", err) } @@ -174,15 +196,13 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName collaboration.UserID = 0 } - // Remove old team-repository relations. if oldOwner.IsOrganization() { + // Remove old team-repository relations. if err := organization.RemoveOrgRepo(ctx, oldOwner.ID, repo.ID); err != nil { return fmt.Errorf("removeOrgRepo: %w", err) } - } - // Remove project's issues that belong to old organization's projects - if oldOwner.IsOrganization() { + // Remove project's issues that belong to old organization's projects projects, err := project_model.GetAllProjectsIDsByOwnerIDAndType(ctx, oldOwner.ID, project_model.TypeOrganization) if err != nil { return fmt.Errorf("Unable to find old org projects: %w", err) @@ -225,15 +245,13 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName return fmt.Errorf("watchRepo: %w", err) } - // Remove watch for organization. if oldOwner.IsOrganization() { + // Remove watch for organization. if err := repo_model.WatchRepo(ctx, oldOwner, repo, false); err != nil { return fmt.Errorf("watchRepo [false]: %w", err) } - } - // Delete labels that belong to the old organization and comments that added these labels - if oldOwner.IsOrganization() { + // Delete labels that belong to the old organization and comments that added these labels if _, err := sess.Exec(`DELETE FROM issue_label WHERE issue_label.id IN ( SELECT il_too.id FROM ( SELECT il_too_too.id @@ -261,7 +279,6 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName // Rename remote repository to new path and delete local copy. dir := user_model.UserPath(newOwner.Name) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { return fmt.Errorf("Failed to create dir %s: %w", dir, err) } @@ -273,7 +290,6 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName // Rename remote wiki repository to new path and delete local copy. wikiPath := repo_model.WikiPath(oldOwner.Name, repo.Name) - if isExist, err := util.IsExist(wikiPath); err != nil { log.Error("Unable to check if %s exists. Error: %v", wikiPath, err) return err @@ -288,7 +304,7 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName return fmt.Errorf("deleteRepositoryTransfer: %w", err) } repo.Status = repo_model.RepositoryReady - if err := repo_model.UpdateRepositoryCols(ctx, repo, "status"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "status"); err != nil { return err } @@ -301,6 +317,17 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName return fmt.Errorf("repo_model.NewRedirect: %w", err) } + newRepo, err := repo_model.GetRepositoryByID(ctx, repo.ID) + if err != nil { + return err + } + + for _, team := range teams { + if err := addRepositoryToTeam(ctx, team, newRepo); err != nil { + return err + } + } + return committer.Commit() } @@ -321,13 +348,13 @@ func changeRepositoryName(ctx context.Context, repo *repo_model.Repository, newR return fmt.Errorf("IsRepositoryExist: %w", err) } else if has { return repo_model.ErrRepoAlreadyExist{ - Uname: repo.Owner.Name, + Uname: repo.OwnerName, Name: newRepoName, } } - newRepoPath := repo_model.RepoPath(repo.Owner.Name, newRepoName) - if err = util.Rename(repo.RepoPath(), newRepoPath); err != nil { + if err = gitrepo.RenameRepository(ctx, repo, + repo_model.StorageRepo(repo_model.RelativePath(repo.OwnerName, newRepoName))); err != nil { return fmt.Errorf("rename repository directory: %w", err) } @@ -343,17 +370,9 @@ func changeRepositoryName(ctx context.Context, repo *repo_model.Repository, newR } } - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - if err := repo_model.NewRedirect(ctx, repo.Owner.ID, repo.ID, oldRepoName, newRepoName); err != nil { - return err - } - - return committer.Commit() + return db.WithTx(ctx, func(ctx context.Context) error { + return repo_model.NewRedirect(ctx, repo.Owner.ID, repo.ID, oldRepoName, newRepoName) + }) } // ChangeRepositoryName changes all corresponding setting from old repository name to new one. @@ -387,70 +406,147 @@ func ChangeRepositoryName(ctx context.Context, doer *user_model.User, repo *repo // StartRepositoryTransfer transfer a repo from one owner to a new one. // it make repository into pending transfer state, if doer can not create repo for new owner. func StartRepositoryTransfer(ctx context.Context, doer, newOwner *user_model.User, repo *repo_model.Repository, teams []*organization.Team) error { + releaser, err := globallock.Lock(ctx, getRepoWorkingLockKey(repo.ID)) + if err != nil { + return fmt.Errorf("lock.Lock: %w", err) + } + defer releaser() + if err := repo_model.TestRepositoryReadyForTransfer(repo.Status); err != nil { return err } - // Admin is always allowed to transfer || user transfer repo back to his account - if doer.IsAdmin || doer.ID == newOwner.ID { - return TransferOwnership(ctx, doer, newOwner, repo, teams) + if !doer.CanForkRepoIn(newOwner) { + limit := util.Iif(newOwner.MaxRepoCreation >= 0, newOwner.MaxRepoCreation, setting.Repository.MaxCreationLimit) + return LimitReachedError{Limit: limit} } - if user_model.IsUserBlockedBy(ctx, doer, newOwner.ID) { - return user_model.ErrBlockedUser - } + var isDirectTransfer bool + oldOwnerName := repo.OwnerName - // If new owner is an org and user can create repos he can transfer directly too - if newOwner.IsOrganization() { - allowed, err := organization.CanCreateOrgRepo(ctx, newOwner.ID, doer.ID) - if err != nil { - return err + if err := db.WithTx(ctx, func(ctx context.Context) error { + // Admin is always allowed to transfer || user transfer repo back to his account, + // then it will transfer directly without acceptance. + if doer.IsAdmin || doer.ID == newOwner.ID { + isDirectTransfer = true + return transferOwnership(ctx, doer, newOwner.Name, repo, teams) } - if allowed { - return TransferOwnership(ctx, doer, newOwner, repo, teams) + + if user_model.IsUserBlockedBy(ctx, doer, newOwner.ID) { + return user_model.ErrBlockedUser } - } - // In case the new owner would not have sufficient access to the repo, give access rights for read - hasAccess, err := access_model.HasAnyUnitAccess(ctx, newOwner.ID, repo) - if err != nil { - return err - } - if !hasAccess { - if err := AddOrUpdateCollaborator(ctx, repo, newOwner, perm.AccessModeRead); err != nil { + // If new owner is an org and user can create repos he can transfer directly too + if newOwner.IsOrganization() { + allowed, err := organization.CanCreateOrgRepo(ctx, newOwner.ID, doer.ID) + if err != nil { + return err + } + if allowed { + isDirectTransfer = true + return transferOwnership(ctx, doer, newOwner.Name, repo, teams) + } + } + + // In case the new owner would not have sufficient access to the repo, give access rights for read + hasAccess, err := access_model.HasAnyUnitAccess(ctx, newOwner.ID, repo) + if err != nil { return err } - } + if !hasAccess { + if err := AddOrUpdateCollaborator(ctx, repo, newOwner, perm.AccessModeRead); err != nil { + return err + } + } - // Make repo as pending for transfer - repo.Status = repo_model.RepositoryPendingTransfer - if err := repo_model.CreatePendingRepositoryTransfer(ctx, doer, newOwner, repo.ID, teams); err != nil { + // Make repo as pending for transfer + repo.Status = repo_model.RepositoryPendingTransfer + return repo_model.CreatePendingRepositoryTransfer(ctx, doer, newOwner, repo.ID, teams) + }); err != nil { return err } - // notify users who are able to accept / reject transfer - notify_service.RepoPendingTransfer(ctx, doer, newOwner, repo) + if isDirectTransfer { + notify_service.TransferRepository(ctx, doer, repo, oldOwnerName) + } else { + // notify users who are able to accept / reject transfer + notify_service.RepoPendingTransfer(ctx, doer, newOwner, repo) + } return nil } -// CancelRepositoryTransfer marks the repository as ready and remove pending transfer entry, +// RejectRepositoryTransfer marks the repository as ready and remove pending transfer entry, // thus cancel the transfer process. -func CancelRepositoryTransfer(ctx context.Context, repo *repo_model.Repository) error { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err +// The accepter can reject the transfer. +func RejectRepositoryTransfer(ctx context.Context, repo *repo_model.Repository, doer *user_model.User) error { + return db.WithTx(ctx, func(ctx context.Context) error { + repoTransfer, err := repo_model.GetPendingRepositoryTransfer(ctx, repo) + if err != nil { + return err + } + + if err := repoTransfer.LoadAttributes(ctx); err != nil { + return err + } + + if !repoTransfer.CanUserAcceptOrRejectTransfer(ctx, doer) { + return util.ErrPermissionDenied + } + + repo.Status = repo_model.RepositoryReady + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "status"); err != nil { + return err + } + + return repo_model.DeleteRepositoryTransfer(ctx, repo.ID) + }) +} + +func canUserCancelTransfer(ctx context.Context, r *repo_model.RepoTransfer, u *user_model.User) bool { + if u.IsAdmin || u.ID == r.DoerID { + return true } - defer committer.Close() - repo.Status = repo_model.RepositoryReady - if err := repo_model.UpdateRepositoryCols(ctx, repo, "status"); err != nil { - return err + if err := r.LoadAttributes(ctx); err != nil { + log.Error("LoadAttributes: %v", err) + return false } - if err := repo_model.DeleteRepositoryTransfer(ctx, repo.ID); err != nil { - return err + if err := r.Repo.LoadOwner(ctx); err != nil { + log.Error("LoadOwner: %v", err) + return false } - return committer.Commit() + if !r.Repo.Owner.IsOrganization() { + return r.Repo.OwnerID == u.ID + } + + perm, err := access_model.GetUserRepoPermission(ctx, r.Repo, u) + if err != nil { + log.Error("GetUserRepoPermission: %v", err) + return false + } + return perm.IsOwner() +} + +// CancelRepositoryTransfer cancels the repository transfer process. The sender or +// the users who have admin permission of the original repository can cancel the transfer +func CancelRepositoryTransfer(ctx context.Context, repoTransfer *repo_model.RepoTransfer, doer *user_model.User) error { + return db.WithTx(ctx, func(ctx context.Context) error { + if err := repoTransfer.LoadAttributes(ctx); err != nil { + return err + } + + if !canUserCancelTransfer(ctx, repoTransfer, doer) { + return util.ErrPermissionDenied + } + + repoTransfer.Repo.Status = repo_model.RepositoryReady + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repoTransfer.Repo, "status"); err != nil { + return err + } + + return repo_model.DeleteRepositoryTransfer(ctx, repoTransfer.RepoID) + }) } diff --git a/services/repository/transfer_test.go b/services/repository/transfer_test.go index 91722fb8ae..80a073e9f9 100644 --- a/services/repository/transfer_test.go +++ b/services/repository/transfer_test.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package repository @@ -14,11 +14,14 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/feed" notify_service "code.gitea.io/gitea/services/notify" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var notifySync sync.Once @@ -34,23 +37,26 @@ func TestTransferOwnership(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) - repo.Owner = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}) - assert.NoError(t, TransferOwnership(db.DefaultContext, doer, doer, repo, nil)) + assert.NoError(t, repo.LoadOwner(db.DefaultContext)) + repoTransfer := unittest.AssertExistsAndLoadBean(t, &repo_model.RepoTransfer{ID: 1}) + assert.NoError(t, repoTransfer.LoadAttributes(db.DefaultContext)) + assert.NoError(t, AcceptTransferOwnership(db.DefaultContext, repo, doer)) transferredRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) - assert.EqualValues(t, 2, transferredRepo.OwnerID) + assert.EqualValues(t, 1, transferredRepo.OwnerID) // repo_transfer.yml id=1 + unittest.AssertNotExistsBean(t, &repo_model.RepoTransfer{ID: 1}) exist, err := util.IsExist(repo_model.RepoPath("org3", "repo3")) assert.NoError(t, err) assert.False(t, exist) - exist, err = util.IsExist(repo_model.RepoPath("user2", "repo3")) + exist, err = util.IsExist(repo_model.RepoPath("user1", "repo3")) assert.NoError(t, err) assert.True(t, exist) unittest.AssertExistsAndLoadBean(t, &activities_model.Action{ OpType: activities_model.ActionTransferRepo, - ActUserID: 2, + ActUserID: 1, RepoID: 3, Content: "org3/repo3", }) @@ -61,10 +67,10 @@ func TestTransferOwnership(t *testing.T) { func TestStartRepositoryTransferSetPermission(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) recipient := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 5}) - repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) - repo.Owner = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2}) + assert.NoError(t, repo.LoadOwner(db.DefaultContext)) hasAccess, err := access_model.HasAnyUnitAccess(db.DefaultContext, recipient.ID, repo) assert.NoError(t, err) @@ -82,7 +88,7 @@ func TestStartRepositoryTransferSetPermission(t *testing.T) { func TestRepositoryTransfer(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) transfer, err := repo_model.GetPendingRepositoryTransfer(db.DefaultContext, repo) @@ -90,7 +96,7 @@ func TestRepositoryTransfer(t *testing.T) { assert.NotNil(t, transfer) // Cancel transfer - assert.NoError(t, CancelRepositoryTransfer(db.DefaultContext, repo)) + assert.NoError(t, CancelRepositoryTransfer(db.DefaultContext, transfer, doer)) transfer, err = repo_model.GetPendingRepositoryTransfer(db.DefaultContext, repo) assert.Error(t, err) @@ -113,10 +119,49 @@ func TestRepositoryTransfer(t *testing.T) { assert.Error(t, err) assert.True(t, repo_model.IsErrRepoTransferInProgress(err)) - // Unknown user - err = repo_model.CreatePendingRepositoryTransfer(db.DefaultContext, doer, &user_model.User{ID: 1000, LowerName: "user1000"}, repo.ID, nil) + repo2 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2}) + // Unknown user, transfer non-existent transfer repo id = 2 + err = repo_model.CreatePendingRepositoryTransfer(db.DefaultContext, doer, &user_model.User{ID: 1000, LowerName: "user1000"}, repo2.ID, nil) assert.Error(t, err) - // Cancel transfer - assert.NoError(t, CancelRepositoryTransfer(db.DefaultContext, repo)) + // Reject transfer + err = RejectRepositoryTransfer(db.DefaultContext, repo2, doer) + assert.True(t, repo_model.IsErrNoPendingTransfer(err)) +} + +// Test transfer rejections +func TestRepositoryTransferRejection(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + // Set limit to 0 repositories so no repositories can be transferred + defer test.MockVariableValue(&setting.Repository.MaxCreationLimit, 0)() + + // Admin case + doerAdmin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 5}) + + transfer, err := repo_model.GetPendingRepositoryTransfer(db.DefaultContext, repo) + require.NoError(t, err) + require.NotNil(t, transfer) + require.NoError(t, transfer.LoadRecipient(db.DefaultContext)) + + require.True(t, doerAdmin.CanCreateRepoIn(transfer.Recipient)) // admin is not subject to limits + + // Administrator should not be affected by the limits so transfer should be successful + assert.NoError(t, AcceptTransferOwnership(db.DefaultContext, repo, doerAdmin)) + + // Non admin user case + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 10}) + repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 21}) + + transfer, err = repo_model.GetPendingRepositoryTransfer(db.DefaultContext, repo) + require.NoError(t, err) + require.NotNil(t, transfer) + require.NoError(t, transfer.LoadRecipient(db.DefaultContext)) + + require.False(t, doer.CanCreateRepoIn(transfer.Recipient)) // regular user is subject to limits + + // Cannot accept because of the limit + err = AcceptTransferOwnership(db.DefaultContext, repo, doer) + assert.Error(t, err) + assert.True(t, IsRepositoryLimitReached(err)) } |