diff options
Diffstat (limited to 'services/repository')
45 files changed, 1599 insertions, 1486 deletions
diff --git a/services/repository/adopt.go b/services/repository/adopt.go index b7321156d9..2bd1c55de4 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -16,7 +16,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" @@ -28,18 +27,30 @@ import ( "github.com/gobwas/glob" ) +func deleteFailedAdoptRepository(repoID int64) error { + return db.WithTx(db.DefaultContext, func(ctx context.Context) error { + if err := deleteDBRepository(ctx, repoID); err != nil { + return fmt.Errorf("deleteDBRepository: %w", err) + } + if err := git_model.DeleteRepoBranches(ctx, repoID); err != nil { + return fmt.Errorf("deleteRepoBranches: %w", err) + } + return repo_model.DeleteRepoReleases(ctx, repoID) + }) +} + // AdoptRepository adopts pre-existing repository files for the user/organization. -func AdoptRepository(ctx context.Context, doer, u *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { - if !doer.IsAdmin && !u.CanCreateRepo() { +func AdoptRepository(ctx context.Context, doer, owner *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { + if !doer.CanCreateRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ - Limit: u.MaxRepoCreation, + Limit: owner.MaxRepoCreation, } } repo := &repo_model.Repository{ - OwnerID: u.ID, - Owner: u, - OwnerName: u.Name, + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, Name: opts.Name, LowerName: strings.ToLower(opts.Name), Description: opts.Description, @@ -48,59 +59,52 @@ func AdoptRepository(ctx context.Context, doer, u *user_model.User, opts CreateR IsPrivate: opts.IsPrivate, IsFsckEnabled: !opts.IsMirror, CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, - Status: opts.Status, + Status: repo_model.RepositoryBeingMigrated, IsEmpty: !opts.AutoInit, } - if err := db.WithTx(ctx, func(ctx context.Context) error { - isExist, err := gitrepo.IsRepositoryExist(ctx, repo) + // 1 - create the repository database operations first + err := db.WithTx(ctx, func(ctx context.Context) error { + return createRepositoryInDB(ctx, doer, owner, repo, false) + }) + if err != nil { + return nil, err + } + + // last - clean up if something goes wrong + // WARNING: Don't override all later err with local variables + defer func() { if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) - return err - } - if !isExist { - return repo_model.ErrRepoNotExist{ - OwnerName: u.Name, - Name: repo.Name, + // we can not use the ctx because it maybe canceled or timeout + if errDel := deleteFailedAdoptRepository(repo.ID); errDel != nil { + log.Error("Failed to delete repository %s that could not be adopted: %v", repo.FullName(), errDel) } } + }() - if err := CreateRepositoryByExample(ctx, doer, u, repo, true, false); err != nil { - return err - } - - // Re-fetch the repository from database before updating it (else it would - // override changes that were done earlier with sql) - if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { - return fmt.Errorf("getRepositoryByID: %w", err) - } - return nil - }); err != nil { - return nil, err + // Re-fetch the repository from database before updating it (else it would + // override changes that were done earlier with sql) + if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { + return nil, fmt.Errorf("getRepositoryByID: %w", err) } - if err := func() error { - if err := adoptRepository(ctx, repo, opts.DefaultBranch); err != nil { - return fmt.Errorf("adoptRepository: %w", err) - } + // 2 - adopt the repository from disk + if err = adoptRepository(ctx, repo, opts.DefaultBranch); err != nil { + return nil, fmt.Errorf("adoptRepository: %w", err) + } - if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { - return fmt.Errorf("checkDaemonExportOK: %w", err) - } + // 3 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } - if stdout, _, err := git.NewCommand("update-server-info"). - RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}); err != nil { - log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - return fmt.Errorf("CreateRepository(git update-server-info): %w", err) - } - return nil - }(); err != nil { - if errDel := DeleteRepository(ctx, doer, repo, false /* no notify */); errDel != nil { - log.Error("Failed to delete repository %s that could not be adopted: %v", repo.FullName(), errDel) - } - return nil, err + // 4 - update repository status + repo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } - notify_service.AdoptRepository(ctx, doer, u, repo) + + notify_service.AdoptRepository(ctx, doer, owner, repo) return repo, nil } @@ -192,8 +196,13 @@ func adoptRepository(ctx context.Context, repo *repo_model.Repository, defaultBr return fmt.Errorf("setDefaultBranch: %w", err) } } - if err = repo_module.UpdateRepository(ctx, repo, false); err != nil { - return fmt.Errorf("updateRepository: %w", err) + + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_empty", "default_branch"); err != nil { + return fmt.Errorf("UpdateRepositoryCols: %w", err) + } + + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) } return nil @@ -256,7 +265,7 @@ func checkUnadoptedRepositories(ctx context.Context, userName string, repoNamesT } return err } - repos, _, err := repo_model.GetUserRepositories(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.GetUserRepositories(ctx, repo_model.SearchRepoOptions{ Actor: ctxUser, Private: true, ListOptions: db.ListOptions{ diff --git a/services/repository/adopt_test.go b/services/repository/adopt_test.go index 123cedc1f2..86f586c748 100644 --- a/services/repository/adopt_test.go +++ b/services/repository/adopt_test.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) @@ -28,7 +29,7 @@ func TestCheckUnadoptedRepositories_Add(t *testing.T) { } total := 30 - for i := 0; i < total; i++ { + for range total { unadopted.add("something") } @@ -71,7 +72,7 @@ func TestListUnadoptedRepositories_ListOptions(t *testing.T) { username := "user2" unadoptedList := []string{path.Join(username, "unadopted1"), path.Join(username, "unadopted2")} for _, unadopted := range unadoptedList { - _ = os.Mkdir(path.Join(setting.RepoRootPath, unadopted+".git"), 0o755) + _ = os.Mkdir(filepath.Join(setting.RepoRootPath, unadopted+".git"), 0o755) } opts := db.ListOptions{Page: 1, PageSize: 1} @@ -89,10 +90,36 @@ func TestListUnadoptedRepositories_ListOptions(t *testing.T) { func TestAdoptRepository(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - assert.NoError(t, unittest.SyncDirs(filepath.Join(setting.RepoRootPath, "user2", "repo1.git"), filepath.Join(setting.RepoRootPath, "user2", "test-adopt.git"))) + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - _, err := AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"}) + + // a successful adopt + destDir := filepath.Join(setting.RepoRootPath, user2.Name, "test-adopt.git") + assert.NoError(t, unittest.SyncDirs(filepath.Join(setting.RepoRootPath, user2.Name, "repo1.git"), destDir)) + + adoptedRepo, err := AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"}) assert.NoError(t, err) repoTestAdopt := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: "test-adopt"}) assert.Equal(t, "sha1", repoTestAdopt.ObjectFormatName) + + // just delete the adopted repo's db records + err = deleteFailedAdoptRepository(adoptedRepo.ID) + assert.NoError(t, err) + + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: "test-adopt"}) + + // a failed adopt because some mock data + // remove the hooks directory and create a file so that we cannot create the hooks successfully + _ = os.RemoveAll(filepath.Join(destDir, "hooks", "update.d")) + assert.NoError(t, os.WriteFile(filepath.Join(destDir, "hooks", "update.d"), []byte("tests"), os.ModePerm)) + + adoptedRepo, err = AdoptRepository(db.DefaultContext, user2, user2, CreateRepoOptions{Name: "test-adopt"}) + assert.Error(t, err) + assert.Nil(t, adoptedRepo) + + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: "test-adopt"}) + + exist, err := util.IsExist(repo_model.RepoPath(user2.Name, "test-adopt")) + assert.NoError(t, err) + assert.True(t, exist) // the repository should be still in the disk } diff --git a/services/repository/archiver/archiver.go b/services/repository/archiver/archiver.go index d39acc080d..a657e3884c 100644 --- a/services/repository/archiver/archiver.go +++ b/services/repository/archiver/archiver.go @@ -44,7 +44,7 @@ type ErrUnknownArchiveFormat struct { // Error implements error func (err ErrUnknownArchiveFormat) Error() string { - return fmt.Sprintf("unknown format: %s", err.RequestNameType) + return "unknown format: " + err.RequestNameType } // Is implements error @@ -60,7 +60,7 @@ type RepoRefNotFoundError struct { // Error implements error. func (e RepoRefNotFoundError) Error() string { - return fmt.Sprintf("unrecognized repository reference: %s", e.RefShortName) + return "unrecognized repository reference: " + e.RefShortName } func (e RepoRefNotFoundError) Is(err error) bool { diff --git a/services/repository/archiver/archiver_test.go b/services/repository/archiver/archiver_test.go index 522f90558a..87324ad38c 100644 --- a/services/repository/archiver/archiver_test.go +++ b/services/repository/archiver/archiver_test.go @@ -33,7 +33,7 @@ func TestArchive_Basic(t *testing.T) { bogusReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) - assert.EqualValues(t, firstCommit+".zip", bogusReq.GetArchiveName()) + assert.Equal(t, firstCommit+".zip", bogusReq.GetArchiveName()) // Check a series of bogus requests. // Step 1, valid commit with a bad extension. @@ -54,12 +54,12 @@ func TestArchive_Basic(t *testing.T) { bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "master.zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) - assert.EqualValues(t, "master.zip", bogusReq.GetArchiveName()) + assert.Equal(t, "master.zip", bogusReq.GetArchiveName()) bogusReq, err = NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, "test/archive.zip") assert.NoError(t, err) assert.NotNil(t, bogusReq) - assert.EqualValues(t, "test-archive.zip", bogusReq.GetArchiveName()) + assert.Equal(t, "test-archive.zip", bogusReq.GetArchiveName()) // Now two valid requests, firstCommit with valid extensions. zipReq, err := NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, firstCommit+".zip") diff --git a/services/repository/avatar.go b/services/repository/avatar.go index 15e51d4a25..998ac42230 100644 --- a/services/repository/avatar.go +++ b/services/repository/avatar.go @@ -29,35 +29,30 @@ func UploadAvatar(ctx context.Context, repo *repo_model.Repository, data []byte) return nil } - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - oldAvatarPath := repo.CustomAvatarRelativePath() - - // Users can upload the same image to other repo - prefix it with ID - // Then repo will be removed - only it avatar file will be removed - repo.Avatar = newAvatar - if err := repo_model.UpdateRepositoryCols(ctx, repo, "avatar"); err != nil { - return fmt.Errorf("UploadAvatar: Update repository avatar: %w", err) - } - - if err := storage.SaveFrom(storage.RepoAvatars, repo.CustomAvatarRelativePath(), func(w io.Writer) error { - _, err := w.Write(avatarData) - return err - }); err != nil { - return fmt.Errorf("UploadAvatar %s failed: Failed to remove old repo avatar %s: %w", repo.RepoPath(), newAvatar, err) - } + return db.WithTx(ctx, func(ctx context.Context) error { + oldAvatarPath := repo.CustomAvatarRelativePath() + + // Users can upload the same image to other repo - prefix it with ID + // Then repo will be removed - only it avatar file will be removed + repo.Avatar = newAvatar + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "avatar"); err != nil { + return fmt.Errorf("UploadAvatar: Update repository avatar: %w", err) + } - if len(oldAvatarPath) > 0 { - if err := storage.RepoAvatars.Delete(oldAvatarPath); err != nil { - return fmt.Errorf("UploadAvatar: Failed to remove old repo avatar %s: %w", oldAvatarPath, err) + if err := storage.SaveFrom(storage.RepoAvatars, repo.CustomAvatarRelativePath(), func(w io.Writer) error { + _, err := w.Write(avatarData) + return err + }); err != nil { + return fmt.Errorf("UploadAvatar %s failed: Failed to remove old repo avatar %s: %w", repo.RepoPath(), newAvatar, err) } - } - return committer.Commit() + if len(oldAvatarPath) > 0 { + if err := storage.RepoAvatars.Delete(oldAvatarPath); err != nil { + return fmt.Errorf("UploadAvatar: Failed to remove old repo avatar %s: %w", oldAvatarPath, err) + } + } + return nil + }) } // DeleteAvatar deletes the repos's custom avatar. @@ -70,22 +65,17 @@ func DeleteAvatar(ctx context.Context, repo *repo_model.Repository) error { avatarPath := repo.CustomAvatarRelativePath() log.Trace("DeleteAvatar[%d]: %s", repo.ID, avatarPath) - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - repo.Avatar = "" - if err := repo_model.UpdateRepositoryCols(ctx, repo, "avatar"); err != nil { - return fmt.Errorf("DeleteAvatar: Update repository avatar: %w", err) - } - - if err := storage.RepoAvatars.Delete(avatarPath); err != nil { - return fmt.Errorf("DeleteAvatar: Failed to remove %s: %w", avatarPath, err) - } + return db.WithTx(ctx, func(ctx context.Context) error { + repo.Avatar = "" + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "avatar"); err != nil { + return fmt.Errorf("DeleteAvatar: Update repository avatar: %w", err) + } - return committer.Commit() + if err := storage.RepoAvatars.Delete(avatarPath); err != nil { + return fmt.Errorf("DeleteAvatar: Failed to remove %s: %w", avatarPath, err) + } + return nil + }) } // RemoveRandomAvatars removes the randomly generated avatars that were created for repositories @@ -112,5 +102,5 @@ func generateAvatar(ctx context.Context, templateRepo, generateRepo *repo_model. return err } - return repo_model.UpdateRepositoryCols(ctx, generateRepo, "avatar") + return repo_model.UpdateRepositoryColsNoAutoTime(ctx, generateRepo, "avatar") } diff --git a/services/repository/avatar_test.go b/services/repository/avatar_test.go index bea820e85f..2dc5173eec 100644 --- a/services/repository/avatar_test.go +++ b/services/repository/avatar_test.go @@ -59,7 +59,7 @@ func TestDeleteAvatar(t *testing.T) { err = DeleteAvatar(db.DefaultContext, repo) assert.NoError(t, err) - assert.Equal(t, "", repo.Avatar) + assert.Empty(t, repo.Avatar) } func TestGenerateAvatar(t *testing.T) { diff --git a/services/repository/branch.go b/services/repository/branch.go index 8804778bd5..6e0065b277 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -233,7 +233,7 @@ func loadOneBranch(ctx context.Context, repo *repo_model.Repository, dbBranch *g defer baseGitRepo.Close() repoIDToGitRepo[pr.BaseRepoID] = baseGitRepo } - pullCommit, err := baseGitRepo.GetRefCommitID(pr.GetGitRefName()) + pullCommit, err := baseGitRepo.GetRefCommitID(pr.GetGitHeadRefName()) if err != nil && !git.IsErrNotExist(err) { return nil, fmt.Errorf("GetBranchCommitID: %v", err) } @@ -303,7 +303,7 @@ func SyncBranchesToDB(ctx context.Context, repoID, pusherID int64, branchNames, // For other batches, it will hit optimization 4. if len(branchNames) != len(commitIDs) { - return fmt.Errorf("branchNames and commitIDs length not match") + return errors.New("branchNames and commitIDs length not match") } return db.WithTx(ctx, func(ctx context.Context) error { @@ -663,6 +663,11 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, newB } } + // clear divergence cache + if err := DelRepoDivergenceFromCache(ctx, repo.ID); err != nil { + log.Error("DelRepoDivergenceFromCache: %v", err) + } + notify_service.ChangeDefaultBranch(ctx, repo) return nil diff --git a/services/repository/check.go b/services/repository/check.go index b475fbc487..ffcd5ac749 100644 --- a/services/repository/check.go +++ b/services/repository/check.go @@ -162,7 +162,7 @@ func DeleteMissingRepositories(ctx context.Context, doer *user_model.User) error default: } log.Trace("Deleting %d/%d...", repo.OwnerID, repo.ID) - if err := DeleteRepositoryDirectly(ctx, doer, repo.ID); err != nil { + if err := DeleteRepositoryDirectly(ctx, repo.ID); err != nil { log.Error("Failed to DeleteRepository %-v: Error: %v", repo, err) if err2 := system_model.CreateRepositoryNotice("Failed to DeleteRepository %s [%d]: Error: %v", repo.FullName(), repo.ID, err); err2 != nil { log.Error("CreateRepositoryNotice: %v", err) diff --git a/services/repository/collaboration.go b/services/repository/collaboration.go index b5fc523623..53b3c2e203 100644 --- a/services/repository/collaboration.go +++ b/services/repository/collaboration.go @@ -71,40 +71,32 @@ func DeleteCollaboration(ctx context.Context, repo *repo_model.Repository, colla UserID: collaborator.ID, } - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - if has, err := db.GetEngine(ctx).Delete(collaboration); err != nil { - return err - } else if has == 0 { - return committer.Commit() - } - - if err := repo.LoadOwner(ctx); err != nil { - return err - } + return db.WithTx(ctx, func(ctx context.Context) error { + if has, err := db.GetEngine(ctx).Delete(collaboration); err != nil { + return err + } else if has == 0 { + return nil + } - if err = access_model.RecalculateAccesses(ctx, repo); err != nil { - return err - } + if err := repo.LoadOwner(ctx); err != nil { + return err + } - if err = repo_model.WatchRepo(ctx, collaborator, repo, false); err != nil { - return err - } + if err = access_model.RecalculateAccesses(ctx, repo); err != nil { + return err + } - if err = ReconsiderWatches(ctx, repo, collaborator); err != nil { - return err - } + if err = repo_model.WatchRepo(ctx, collaborator, repo, false); err != nil { + return err + } - // Unassign a user from any issue (s)he has been assigned to in the repository - if err := ReconsiderRepoIssuesAssignee(ctx, repo, collaborator); err != nil { - return err - } + if err = ReconsiderWatches(ctx, repo, collaborator); err != nil { + return err + } - return committer.Commit() + // Unassign a user from any issue (s)he has been assigned to in the repository + return ReconsiderRepoIssuesAssignee(ctx, repo, collaborator) + }) } func ReconsiderRepoIssuesAssignee(ctx context.Context, repo *repo_model.Repository, user *user_model.User) error { diff --git a/services/repository/commitstatus/commitstatus.go b/services/repository/commitstatus/commitstatus.go index f369a303e6..fa7a89882a 100644 --- a/services/repository/commitstatus/commitstatus.go +++ b/services/repository/commitstatus/commitstatus.go @@ -14,17 +14,17 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/commitstatus" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" - api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/services/notify" ) func getCacheKey(repoID int64, brancheName string) string { - hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%d:%s", repoID, brancheName))) + hashBytes := sha256.Sum256(fmt.Appendf(nil, "%d:%s", repoID, brancheName)) return fmt.Sprintf("commit_status:%x", hashBytes) } @@ -47,7 +47,7 @@ func getCommitStatusCache(repoID int64, branchName string) *commitStatusCacheVal return nil } -func updateCommitStatusCache(repoID int64, branchName string, state api.CommitStatusState, targetURL string) error { +func updateCommitStatusCache(repoID int64, branchName string, state commitstatus.CommitStatusState, targetURL string) error { c := cache.GetCache() bs, err := json.Marshal(commitStatusCacheValue{ State: state.String(), @@ -127,7 +127,7 @@ func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Rep for i, repo := range repos { if cv := getCommitStatusCache(repo.ID, repo.DefaultBranch); cv != nil { results[i] = &git_model.CommitStatus{ - State: api.CommitStatusState(cv.State), + State: commitstatus.CommitStatusState(cv.State), TargetURL: cv.TargetURL, } } else { diff --git a/services/repository/contributors_graph_test.go b/services/repository/contributors_graph_test.go index 6db93f6a64..7d32b1c931 100644 --- a/services/repository/contributors_graph_test.go +++ b/services/repository/contributors_graph_test.go @@ -38,14 +38,14 @@ func TestRepository_ContributorsGraph(t *testing.T) { keys = append(keys, k) } slices.Sort(keys) - assert.EqualValues(t, []string{ + assert.Equal(t, []string{ "ethantkoenig@gmail.com", "jimmy.praet@telenet.be", "jon@allspice.io", "total", // generated summary }, keys) - assert.EqualValues(t, &ContributorData{ + assert.Equal(t, &ContributorData{ Name: "Ethan Koenig", AvatarLink: "/assets/img/avatar_default.png", TotalCommits: 1, @@ -58,7 +58,7 @@ func TestRepository_ContributorsGraph(t *testing.T) { }, }, }, data["ethantkoenig@gmail.com"]) - assert.EqualValues(t, &ContributorData{ + assert.Equal(t, &ContributorData{ Name: "Total", AvatarLink: "", TotalCommits: 3, diff --git a/services/repository/create.go b/services/repository/create.go index 1a6a68b35a..bed02e5d7e 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/models/perm" access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" + system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" @@ -28,7 +29,6 @@ import ( "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/templates/vars" - "code.gitea.io/gitea/modules/util" ) // CreateRepoOptions contains the create repository options @@ -100,8 +100,8 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir // .gitignore if len(opts.Gitignores) > 0 { var buf bytes.Buffer - names := strings.Split(opts.Gitignores, ",") - for _, name := range names { + names := strings.SplitSeq(opts.Gitignores, ",") + for name := range names { data, err = options.Gitignore(name) if err != nil { return fmt.Errorf("GetRepoInitFile[%s]: %w", name, err) @@ -140,21 +140,20 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir // InitRepository initializes README and .gitignore if needed. func initRepository(ctx context.Context, u *user_model.User, repo *repo_model.Repository, opts CreateRepoOptions) (err error) { - if err = repo_module.CheckInitRepository(ctx, repo); err != nil { - return err + // Init git bare new repository. + if err = git.InitRepository(ctx, repo.RepoPath(), true, repo.ObjectFormatName); err != nil { + return fmt.Errorf("git.InitRepository: %w", err) + } else if err = gitrepo.CreateDelegateHooks(ctx, repo); err != nil { + return fmt.Errorf("createDelegateHooks: %w", err) } // Initialize repository according to user's choice. if opts.AutoInit { - tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name) + tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("repos-" + repo.Name) if err != nil { - return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.FullName(), err) + return fmt.Errorf("failed to create temp dir for repository %s: %w", repo.FullName(), err) } - defer func() { - if err := util.RemoveAll(tmpDir); err != nil { - log.Warn("Unable to remove temporary directory: %s: Error: %v", tmpDir, err) - } - }() + defer cleanup() if err = prepareRepoCommit(ctx, repo, tmpDir, opts); err != nil { return fmt.Errorf("prepareRepoCommit: %w", err) @@ -192,18 +191,25 @@ func initRepository(ctx context.Context, u *user_model.User, repo *repo_model.Re } } - if err = UpdateRepository(ctx, repo, false); err != nil { + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_empty", "default_branch", "default_wiki_branch"); err != nil { return fmt.Errorf("updateRepository: %w", err) } + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + return nil } // CreateRepositoryDirectly creates a repository for the user/organization. -func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { - if !doer.IsAdmin && !u.CanCreateRepo() { +// if needsUpdateToReady is true, it will update the repository status to ready when success +func CreateRepositoryDirectly(ctx context.Context, doer, owner *user_model.User, + opts CreateRepoOptions, needsUpdateToReady bool, +) (*repo_model.Repository, error) { + if !doer.CanCreateRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ - Limit: u.MaxRepoCreation, + Limit: owner.MaxRepoCreation, } } @@ -223,9 +229,9 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt } repo := &repo_model.Repository{ - OwnerID: u.ID, - Owner: u, - OwnerName: u.Name, + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, Name: opts.Name, LowerName: strings.ToLower(opts.Name), Description: opts.Description, @@ -244,100 +250,91 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt ObjectFormatName: opts.ObjectFormatName, } - var rollbackRepo *repo_model.Repository - - if err := db.WithTx(ctx, func(ctx context.Context) error { - if err := CreateRepositoryByExample(ctx, doer, u, repo, false, false); err != nil { - return err - } - - // No need for init mirror. - if opts.IsMirror { - return nil - } + // 1 - create the repository database operations first + err := db.WithTx(ctx, func(ctx context.Context) error { + return createRepositoryInDB(ctx, doer, owner, repo, false) + }) + if err != nil { + return nil, err + } - isExist, err := gitrepo.IsRepositoryExist(ctx, repo) + // last - clean up if something goes wrong + // WARNING: Don't override all later err with local variables + defer func() { if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) - return err - } - if isExist { - // repo already exists - We have two or three options. - // 1. We fail stating that the directory exists - // 2. We create the db repository to go with this data and adopt the git repo - // 3. We delete it and start afresh - // - // Previously Gitea would just delete and start afresh - this was naughty. - // So we will now fail and delegate to other functionality to adopt or delete - log.Error("Files already exist in %s and we are not going to adopt or delete.", repo.FullName()) - return repo_model.ErrRepoFilesAlreadyExist{ - Uname: u.Name, - Name: repo.Name, - } + // we can not use the ctx because it maybe canceled or timeout + cleanupRepository(repo.ID) } + }() - if err = initRepository(ctx, doer, repo, opts); err != nil { - if err2 := gitrepo.DeleteRepository(ctx, repo); err2 != nil { - log.Error("initRepository: %v", err) - return fmt.Errorf( - "delete repo directory %s/%s failed(2): %v", u.Name, repo.Name, err2) - } - return fmt.Errorf("initRepository: %w", err) - } + // No need for init mirror. + if opts.IsMirror { + return repo, nil + } - // Initialize Issue Labels if selected - if len(opts.IssueLabels) > 0 { - if err = repo_module.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { - rollbackRepo = repo - rollbackRepo.OwnerID = u.ID - return fmt.Errorf("InitializeLabels: %w", err) - } + // 2 - check whether the repository with the same storage exists + var isExist bool + isExist, err = gitrepo.IsRepositoryExist(ctx, repo) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) + return nil, err + } + if isExist { + log.Error("Files already exist in %s and we are not going to adopt or delete.", repo.FullName()) + // Don't return directly, we need err in defer to cleanupRepository + err = repo_model.ErrRepoFilesAlreadyExist{ + Uname: repo.OwnerName, + Name: repo.Name, } + return nil, err + } - if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { - return fmt.Errorf("checkDaemonExportOK: %w", err) - } + // 3 - init git repository in storage + if err = initRepository(ctx, doer, repo, opts); err != nil { + return nil, fmt.Errorf("initRepository: %w", err) + } - if stdout, _, err := git.NewCommand("update-server-info"). - RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}); err != nil { - log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - rollbackRepo = repo - rollbackRepo.OwnerID = u.ID - return fmt.Errorf("CreateRepository(git update-server-info): %w", err) + // 4 - Initialize Issue Labels if selected + if len(opts.IssueLabels) > 0 { + if err = repo_module.InitializeLabels(ctx, repo.ID, opts.IssueLabels, false); err != nil { + return nil, fmt.Errorf("InitializeLabels: %w", err) } + } - // update licenses - var licenses []string - if len(opts.License) > 0 { - licenses = append(licenses, opts.License) + // 5 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } - stdout, _, err := git.NewCommand("rev-parse", "HEAD").RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) - if err != nil { - log.Error("CreateRepository(git rev-parse HEAD) in %v: Stdout: %s\nError: %v", repo, stdout, err) - rollbackRepo = repo - rollbackRepo.OwnerID = u.ID - return fmt.Errorf("CreateRepository(git rev-parse HEAD): %w", err) - } - if err := repo_model.UpdateRepoLicenses(ctx, repo, stdout, licenses); err != nil { - return err - } + // 6 - update licenses + var licenses []string + if len(opts.License) > 0 { + licenses = append(licenses, opts.License) + + var stdout string + stdout, _, err = git.NewCommand("rev-parse", "HEAD").RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}) + if err != nil { + log.Error("CreateRepository(git rev-parse HEAD) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return nil, fmt.Errorf("CreateRepository(git rev-parse HEAD): %w", err) } - return nil - }); err != nil { - if rollbackRepo != nil { - if errDelete := DeleteRepositoryDirectly(ctx, doer, rollbackRepo.ID); errDelete != nil { - log.Error("Rollback deleteRepository: %v", errDelete) - } + if err = repo_model.UpdateRepoLicenses(ctx, repo, stdout, licenses); err != nil { + return nil, err } + } - return nil, err + // 7 - update repository status to be ready + if needsUpdateToReady { + repo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) + } } return repo, nil } -// CreateRepositoryByExample creates a repository for the user/organization. -func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository, overwriteOrAdopt, isFork bool) (err error) { +// createRepositoryInDB creates a repository for the user/organization. +func createRepositoryInDB(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository, isFork bool) (err error) { if err = repo_model.IsUsableRepoName(repo.Name); err != nil { return err } @@ -352,19 +349,6 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re } } - isExist, err := gitrepo.IsRepositoryExist(ctx, repo) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) - return err - } - if !overwriteOrAdopt && isExist { - log.Error("Files already exist in %s and we are not going to adopt or delete.", repo.FullName()) - return repo_model.ErrRepoFilesAlreadyExist{ - Uname: u.Name, - Name: repo.Name, - } - } - if err = db.Insert(ctx, repo); err != nil { return err } @@ -384,7 +368,8 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re } units := make([]repo_model.RepoUnit, 0, len(defaultUnits)) for _, tp := range defaultUnits { - if tp == unit.TypeIssues { + switch tp { + case unit.TypeIssues: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, @@ -394,7 +379,7 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re EnableDependencies: setting.Service.DefaultEnableDependencies, }, }) - } else if tp == unit.TypePullRequests { + case unit.TypePullRequests: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, @@ -404,13 +389,13 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re AllowRebaseUpdate: true, }, }) - } else if tp == unit.TypeProjects { + case unit.TypeProjects: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, Config: &repo_model.ProjectsConfig{ProjectsMode: repo_model.ProjectsModeAll}, }) - } else { + default: units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, Type: tp, @@ -472,3 +457,26 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re return nil } + +func cleanupRepository(repoID int64) { + if errDelete := DeleteRepositoryDirectly(db.DefaultContext, repoID); errDelete != nil { + log.Error("cleanupRepository failed: %v", errDelete) + // add system notice + if err := system_model.CreateRepositoryNotice("DeleteRepositoryDirectly failed when cleanup repository: %v", errDelete); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + } +} + +func updateGitRepoAfterCreate(ctx context.Context, repo *repo_model.Repository) error { + if err := checkDaemonExportOK(ctx, repo); err != nil { + return fmt.Errorf("checkDaemonExportOK: %w", err) + } + + if stdout, _, err := git.NewCommand("update-server-info"). + RunStdString(ctx, &git.RunOpts{Dir: repo.RepoPath()}); err != nil { + log.Error("CreateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) + return fmt.Errorf("CreateRepository(git update-server-info): %w", err) + } + return nil +} diff --git a/services/repository/create_test.go b/services/repository/create_test.go new file mode 100644 index 0000000000..fe464c1441 --- /dev/null +++ b/services/repository/create_test.go @@ -0,0 +1,57 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "os" + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/util" + + "github.com/stretchr/testify/assert" +) + +func TestCreateRepositoryDirectly(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + // a successful creating repository + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + createdRepo, err := CreateRepositoryDirectly(git.DefaultContext, user2, user2, CreateRepoOptions{ + Name: "created-repo", + }, true) + assert.NoError(t, err) + assert.NotNil(t, createdRepo) + + exist, err := util.IsExist(repo_model.RepoPath(user2.Name, createdRepo.Name)) + assert.NoError(t, err) + assert.True(t, exist) + + unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: createdRepo.Name}) + + err = DeleteRepositoryDirectly(db.DefaultContext, createdRepo.ID) + assert.NoError(t, err) + + // a failed creating because some mock data + // create the repository directory so that the creation will fail after database record created. + assert.NoError(t, os.MkdirAll(repo_model.RepoPath(user2.Name, createdRepo.Name), os.ModePerm)) + + createdRepo2, err := CreateRepositoryDirectly(db.DefaultContext, user2, user2, CreateRepoOptions{ + Name: "created-repo", + }, true) + assert.Nil(t, createdRepo2) + assert.Error(t, err) + + // assert the cleanup is successful + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: createdRepo.Name}) + + exist, err = util.IsExist(repo_model.RepoPath(user2.Name, createdRepo.Name)) + assert.NoError(t, err) + assert.False(t, exist) +} diff --git a/services/repository/delete.go b/services/repository/delete.go index ff74a20817..c48d6e1d56 100644 --- a/services/repository/delete.go +++ b/services/repository/delete.go @@ -27,14 +27,29 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" + actions_service "code.gitea.io/gitea/services/actions" asymkey_service "code.gitea.io/gitea/services/asymkey" + issue_service "code.gitea.io/gitea/services/issue" "xorm.io/builder" ) +func deleteDBRepository(ctx context.Context, repoID int64) error { + if cnt, err := db.GetEngine(ctx).ID(repoID).Delete(&repo_model.Repository{}); err != nil { + return err + } else if cnt != 1 { + return repo_model.ErrRepoNotExist{ + ID: repoID, + OwnerName: "", + Name: "", + } + } + return nil +} + // DeleteRepository deletes a repository for a user or organization. // make sure if you call this func to close open sessions (sqlite will otherwise get a deadlock) -func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID int64, ignoreOrgTeams ...bool) error { +func DeleteRepositoryDirectly(ctx context.Context, repoID int64, ignoreOrgTeams ...bool) error { ctx, committer, err := db.TxContext(ctx) if err != nil { return err @@ -82,14 +97,8 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID } needRewriteKeysFile := deleted > 0 - if cnt, err := sess.ID(repoID).Delete(&repo_model.Repository{}); err != nil { + if err := deleteDBRepository(ctx, repoID); err != nil { return err - } else if cnt != 1 { - return repo_model.ErrRepoNotExist{ - ID: repoID, - OwnerName: "", - Name: "", - } } if org != nil && org.IsOrganization() { @@ -126,6 +135,14 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID return err } + // CleanupEphemeralRunnersByPickedTaskOfRepo deletes ephemeral global/org/user that have started any task of this repo + // The cannot pick a second task hardening for ephemeral runners expect that task objects remain available until runner deletion + // This method will delete affected ephemeral global/org/user runners + // &actions_model.ActionRunner{RepoID: repoID} does only handle ephemeral repository runners + if err := actions_service.CleanupEphemeralRunnersByPickedTaskOfRepo(ctx, repoID); err != nil { + return fmt.Errorf("cleanupEphemeralRunners: %w", err) + } + if err := db.DeleteBeans(ctx, &access_model.Access{RepoID: repo.ID}, &activities_model.Action{RepoID: repo.ID}, @@ -177,7 +194,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID // Delete Issues and related objects var attachmentPaths []string - if attachmentPaths, err = issues_model.DeleteIssuesByRepoID(ctx, repoID); err != nil { + if attachmentPaths, err = issue_service.DeleteIssuesByRepoID(ctx, repoID); err != nil { return err } @@ -358,7 +375,7 @@ func DeleteRepositoryDirectly(ctx context.Context, doer *user_model.User, repoID // DeleteOwnerRepositoriesDirectly calls DeleteRepositoryDirectly for all repos of the given owner func DeleteOwnerRepositoriesDirectly(ctx context.Context, owner *user_model.User) error { for { - repos, _, err := repo_model.GetUserRepositories(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.GetUserRepositories(ctx, repo_model.SearchRepoOptions{ ListOptions: db.ListOptions{ PageSize: repo_model.RepositoryListDefaultPageSize, Page: 1, @@ -374,7 +391,7 @@ func DeleteOwnerRepositoriesDirectly(ctx context.Context, owner *user_model.User break } for _, repo := range repos { - if err := DeleteRepositoryDirectly(ctx, owner, repo.ID); err != nil { + if err := DeleteRepositoryDirectly(ctx, repo.ID); err != nil { return fmt.Errorf("unable to delete repository %s for %s[%d]. Error: %w", repo.Name, owner.Name, owner.ID, err) } } diff --git a/services/repository/files/cherry_pick.go b/services/repository/files/cherry_pick.go index 0e069fb2ce..6818bb343d 100644 --- a/services/repository/files/cherry_pick.go +++ b/services/repository/files/cherry_pick.go @@ -5,6 +5,7 @@ package files import ( "context" + "errors" "fmt" "strings" @@ -100,7 +101,7 @@ func CherryPick(ctx context.Context, repo *repo_model.Repository, doer *user_mod } if conflict { - return nil, fmt.Errorf("failed to merge due to conflicts") + return nil, errors.New("failed to merge due to conflicts") } treeHash, err := t.WriteTree(ctx) diff --git a/services/repository/files/content.go b/services/repository/files/content.go index 0ab7422ce2..2c1e88bb59 100644 --- a/services/repository/files/content.go +++ b/services/repository/files/content.go @@ -5,17 +5,18 @@ package files import ( "context" - "fmt" + "io" "net/url" "path" "strings" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" ) // ContentType repo content type @@ -23,14 +24,10 @@ type ContentType string // The string representations of different content types const ( - // ContentTypeRegular regular content type (file) - ContentTypeRegular ContentType = "file" - // ContentTypeDir dir content type (dir) - ContentTypeDir ContentType = "dir" - // ContentLink link content type (symlink) - ContentTypeLink ContentType = "symlink" - // ContentTag submodule content type (submodule) - ContentTypeSubmodule ContentType = "submodule" + ContentTypeRegular ContentType = "file" // regular content type (file) + ContentTypeDir ContentType = "dir" // dir content type (dir) + ContentTypeLink ContentType = "symlink" // link content type (symlink) + ContentTypeSubmodule ContentType = "submodule" // submodule content type (submodule) ) // String gets the string of ContentType @@ -38,67 +35,52 @@ func (ct *ContentType) String() string { return string(*ct) } -// GetContentsOrList gets the meta data of a file's contents (*ContentsResponse) if treePath not a tree -// directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag -func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, treePath, ref string) (any, error) { - if repo.IsEmpty { - return make([]any, 0), nil - } - if ref == "" { - ref = repo.DefaultBranch - } - origRef := ref - - // Check that the path given in opts.treePath is valid (not a git path) - cleanTreePath := CleanUploadFileName(treePath) - if cleanTreePath == "" && treePath != "" { - return nil, ErrFilenameInvalid{ - Path: treePath, - } - } - treePath = cleanTreePath - - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return nil, err - } - defer closer.Close() +type GetContentsOrListOptions struct { + TreePath string + IncludeSingleFileContent bool // include the file's content when the tree path is a file + IncludeLfsMetadata bool + IncludeCommitMetadata bool + IncludeCommitMessage bool +} - // Get the commit object for the ref - commit, err := gitRepo.GetCommit(ref) - if err != nil { - return nil, err +// GetContentsOrList gets the metadata of a file's contents (*ContentsResponse) if treePath not a tree +// directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag +func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, opts GetContentsOrListOptions) (ret api.ContentsExtResponse, _ error) { + entry, err := prepareGetContentsEntry(refCommit, &opts.TreePath) + if repo.IsEmpty && opts.TreePath == "" { + return api.ContentsExtResponse{DirContents: make([]*api.ContentsResponse, 0)}, nil } - - entry, err := commit.GetTreeEntryByPath(treePath) if err != nil { - return nil, err + return ret, err } + // get file contents if entry.Type() != "tree" { - return GetContents(ctx, repo, treePath, origRef, false) + ret.FileContents, err = getFileContentsByEntryInternal(ctx, repo, gitRepo, refCommit, entry, opts) + return ret, err } - // We are in a directory, so we return a list of FileContentResponse objects - var fileList []*api.ContentsResponse - - gitTree, err := commit.SubTree(treePath) + // list directory contents + gitTree, err := refCommit.Commit.SubTree(opts.TreePath) if err != nil { - return nil, err + return ret, err } entries, err := gitTree.ListEntries() if err != nil { - return nil, err + return ret, err } + ret.DirContents = make([]*api.ContentsResponse, 0, len(entries)) for _, e := range entries { - subTreePath := path.Join(treePath, e.Name()) - fileContentResponse, err := GetContents(ctx, repo, subTreePath, origRef, true) + subOpts := opts + subOpts.TreePath = path.Join(opts.TreePath, e.Name()) + subOpts.IncludeSingleFileContent = false // never include file content when listing a directory + fileContentResponse, err := GetFileContents(ctx, repo, gitRepo, refCommit, subOpts) if err != nil { - return nil, err + return ret, err } - fileList = append(fileList, fileContentResponse) + ret.DirContents = append(ret.DirContents, fileContentResponse) } - return fileList, nil + return ret, nil } // GetObjectTypeFromTreeEntry check what content is behind it @@ -117,86 +99,96 @@ func GetObjectTypeFromTreeEntry(entry *git.TreeEntry) ContentType { } } -// GetContents gets the meta data on a file's contents. Ref can be a branch, commit or tag -func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref string, forList bool) (*api.ContentsResponse, error) { - if ref == "" { - ref = repo.DefaultBranch - } - origRef := ref - +func prepareGetContentsEntry(refCommit *utils.RefCommit, treePath *string) (*git.TreeEntry, error) { // Check that the path given in opts.treePath is valid (not a git path) - cleanTreePath := CleanUploadFileName(treePath) - if cleanTreePath == "" && treePath != "" { - return nil, ErrFilenameInvalid{ - Path: treePath, - } + cleanTreePath := CleanGitTreePath(*treePath) + if cleanTreePath == "" && *treePath != "" { + return nil, ErrFilenameInvalid{Path: *treePath} } - treePath = cleanTreePath + *treePath = cleanTreePath - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return nil, err + // Only allow safe ref types + refType := refCommit.RefName.RefType() + if refType != git.RefTypeBranch && refType != git.RefTypeTag && refType != git.RefTypeCommit { + return nil, util.NewNotExistErrorf("no commit found for the ref [ref: %s]", refCommit.RefName) } - defer closer.Close() - // Get the commit object for the ref - commit, err := gitRepo.GetCommit(ref) - if err != nil { - return nil, err - } - commitID := commit.ID.String() - if len(ref) >= 4 && strings.HasPrefix(commitID, ref) { - ref = commit.ID.String() - } + return refCommit.Commit.GetTreeEntryByPath(*treePath) +} - entry, err := commit.GetTreeEntryByPath(treePath) +// GetFileContents gets the metadata on a file's contents. Ref can be a branch, commit or tag +func GetFileContents(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, opts GetContentsOrListOptions) (*api.ContentsResponse, error) { + entry, err := prepareGetContentsEntry(refCommit, &opts.TreePath) if err != nil { return nil, err } + return getFileContentsByEntryInternal(ctx, repo, gitRepo, refCommit, entry, opts) +} - refType := gitRepo.GetRefType(ref) - if refType == "invalid" { - return nil, fmt.Errorf("no commit found for the ref [ref: %s]", ref) - } - - selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(treePath) + "?ref=" + url.QueryEscape(origRef)) +func getFileContentsByEntryInternal(_ context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, entry *git.TreeEntry, opts GetContentsOrListOptions) (*api.ContentsResponse, error) { + refType := refCommit.RefName.RefType() + commit := refCommit.Commit + selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(opts.TreePath) + "?ref=" + url.QueryEscape(refCommit.InputRef)) if err != nil { return nil, err } selfURLString := selfURL.String() - err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(ref, refType != git.ObjectCommit), repo.FullName(), commitID) - if err != nil { - return nil, err - } - - lastCommit, err := commit.GetCommitByPath(treePath) - if err != nil { - return nil, err - } - // All content types have these fields in populated contentsResponse := &api.ContentsResponse{ - Name: entry.Name(), - Path: treePath, - SHA: entry.ID.String(), - LastCommitSHA: lastCommit.ID.String(), - Size: entry.Size(), - URL: &selfURLString, + Name: entry.Name(), + Path: opts.TreePath, + SHA: entry.ID.String(), + Size: entry.Size(), + URL: &selfURLString, Links: &api.FileLinksResponse{ Self: &selfURLString, }, } - // Now populate the rest of the ContentsResponse based on entry type + if opts.IncludeCommitMetadata || opts.IncludeCommitMessage { + err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(refCommit.InputRef, refType != git.RefTypeCommit), repo.FullName(), refCommit.CommitID) + if err != nil { + return nil, err + } + + lastCommit, err := refCommit.Commit.GetCommitByPath(opts.TreePath) + if err != nil { + return nil, err + } + + if opts.IncludeCommitMetadata { + contentsResponse.LastCommitSHA = util.ToPointer(lastCommit.ID.String()) + // GitHub doesn't have these fields in the response, but we could follow other similar APIs to name them + // https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits + if lastCommit.Committer != nil { + contentsResponse.LastCommitterDate = util.ToPointer(lastCommit.Committer.When) + } + if lastCommit.Author != nil { + contentsResponse.LastAuthorDate = util.ToPointer(lastCommit.Author.When) + } + } + if opts.IncludeCommitMessage { + contentsResponse.LastCommitMessage = util.ToPointer(lastCommit.Message()) + } + } + + // Now populate the rest of the ContentsResponse based on the entry type if entry.IsRegular() || entry.IsExecutable() { contentsResponse.Type = string(ContentTypeRegular) - if blobResponse, err := GetBlobBySHA(ctx, repo, gitRepo, entry.ID.String()); err != nil { - return nil, err - } else if !forList { - // We don't show the content if we are getting a list of FileContentResponses - contentsResponse.Encoding = &blobResponse.Encoding - contentsResponse.Content = &blobResponse.Content + // if it is listing the repo root dir, don't waste system resources on reading content + if opts.IncludeSingleFileContent { + blobResponse, err := GetBlobBySHA(repo, gitRepo, entry.ID.String()) + if err != nil { + return nil, err + } + contentsResponse.Encoding, contentsResponse.Content = blobResponse.Encoding, blobResponse.Content + contentsResponse.LfsOid, contentsResponse.LfsSize = blobResponse.LfsOid, blobResponse.LfsSize + } else if opts.IncludeLfsMetadata { + contentsResponse.LfsOid, contentsResponse.LfsSize, err = parsePossibleLfsPointerBlob(gitRepo, entry.ID.String()) + if err != nil { + return nil, err + } } } else if entry.IsDir() { contentsResponse.Type = string(ContentTypeDir) @@ -210,7 +202,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref contentsResponse.Target = &targetFromContent } else if entry.IsSubModule() { contentsResponse.Type = string(ContentTypeSubmodule) - submodule, err := commit.GetSubModule(treePath) + submodule, err := commit.GetSubModule(opts.TreePath) if err != nil { return nil, err } @@ -220,7 +212,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref } // Handle links if entry.IsRegular() || entry.IsLink() || entry.IsExecutable() { - downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(opts.TreePath)) if err != nil { return nil, err } @@ -228,7 +220,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref contentsResponse.DownloadURL = &downloadURLString } if !entry.IsSubModule() { - htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(opts.TreePath)) if err != nil { return nil, err } @@ -248,49 +240,59 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref return contentsResponse, nil } -// GetBlobBySHA get the GitBlobResponse of a repository using a sha hash. -func GetBlobBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) { +func GetBlobBySHA(repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) { gitBlob, err := gitRepo.GetBlob(sha) if err != nil { return nil, err } - content := "" - if gitBlob.Size() <= setting.API.DefaultMaxBlobSize { - content, err = gitBlob.GetBlobContentBase64() - if err != nil { - return nil, err - } + ret := &api.GitBlobResponse{ + SHA: gitBlob.ID.String(), + URL: repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()), + Size: gitBlob.Size(), } - return &api.GitBlobResponse{ - SHA: gitBlob.ID.String(), - URL: repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()), - Size: gitBlob.Size(), - Encoding: "base64", - Content: content, - }, nil -} -// TryGetContentLanguage tries to get the (linguist) language of the file content -func TryGetContentLanguage(gitRepo *git.Repository, commitID, treePath string) (string, error) { - indexFilename, worktree, deleteTemporaryFile, err := gitRepo.ReadTreeToTemporaryIndex(commitID) - if err != nil { - return "", err + blobSize := gitBlob.Size() + if blobSize > setting.API.DefaultMaxBlobSize { + return ret, nil } - defer deleteTemporaryFile() + var originContent *strings.Builder + if 0 < blobSize && blobSize < lfs.MetaFileMaxSize { + originContent = &strings.Builder{} + } - filename2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{ - CachedOnly: true, - Attributes: []string{git.AttributeLinguistLanguage, git.AttributeGitlabLanguage}, - Filenames: []string{treePath}, - IndexFile: indexFilename, - WorkTree: worktree, - }) + content, err := gitBlob.GetBlobContentBase64(originContent) if err != nil { - return "", err + return nil, err + } + + ret.Encoding, ret.Content = util.ToPointer("base64"), &content + if originContent != nil { + ret.LfsOid, ret.LfsSize = parsePossibleLfsPointerBuffer(strings.NewReader(originContent.String())) } + return ret, nil +} - language := git.TryReadLanguageAttribute(filename2attribute2info[treePath]) +func parsePossibleLfsPointerBuffer(r io.Reader) (*string, *int64) { + p, _ := lfs.ReadPointer(r) + if p.IsValid() { + return &p.Oid, &p.Size + } + return nil, nil +} - return language.Value(), nil +func parsePossibleLfsPointerBlob(gitRepo *git.Repository, sha string) (*string, *int64, error) { + gitBlob, err := gitRepo.GetBlob(sha) + if err != nil { + return nil, nil, err + } + if gitBlob.Size() > lfs.MetaFileMaxSize { + return nil, nil, nil // not a LFS pointer + } + buf, err := gitBlob.GetBlobContent(lfs.MetaFileMaxSize) + if err != nil { + return nil, nil, err + } + oid, size := parsePossibleLfsPointerBuffer(strings.NewReader(buf)) + return oid, size, nil } diff --git a/services/repository/files/content_test.go b/services/repository/files/content_test.go index 7cb46c0bb6..d72f918074 100644 --- a/services/repository/files/content_test.go +++ b/services/repository/files/content_test.go @@ -7,8 +7,8 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/gitrepo" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/contexttest" _ "code.gitea.io/gitea/models/actions" @@ -20,36 +20,6 @@ func TestMain(m *testing.M) { unittest.MainTest(m) } -func getExpectedReadmeContentsResponse() *api.ContentsResponse { - treePath := "README.md" - sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" - encoding := "base64" - content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" - selfURL := "https://try.gitea.io/api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" - htmlURL := "https://try.gitea.io/user2/repo1/src/branch/master/" + treePath - gitURL := "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/" + sha - downloadURL := "https://try.gitea.io/user2/repo1/raw/branch/master/" + treePath - return &api.ContentsResponse{ - Name: treePath, - Path: treePath, - SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", - LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - Type: "file", - Size: 30, - Encoding: &encoding, - Content: &content, - URL: &selfURL, - HTMLURL: &htmlURL, - GitURL: &gitURL, - DownloadURL: &downloadURL, - Links: &api.FileLinksResponse{ - Self: &selfURL, - GitURL: &gitURL, - HTMLURL: &htmlURL, - }, - } -} - func TestGetContents(t *testing.T) { unittest.PrepareTestEnv(t) ctx, _ := contexttest.MockContext(t, "user2/repo1") @@ -58,195 +28,22 @@ func TestGetContents(t *testing.T) { contexttest.LoadRepoCommit(t, ctx) contexttest.LoadUser(t, ctx, 2) contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - treePath := "README.md" - ref := ctx.Repo.Repository.DefaultBranch - - expectedContentsResponse := getExpectedReadmeContentsResponse() - - t.Run("Get README.md contents with GetContents(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContents(ctx, ctx.Repo.Repository, treePath, ref, false) - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContents(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContents(ctx, ctx.Repo.Repository, treePath, "", false) - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) -} - -func TestGetContentsOrListForDir(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - treePath := "" // root dir - ref := ctx.Repo.Repository.DefaultBranch - - readmeContentsResponse := getExpectedReadmeContentsResponse() - // because will be in a list, doesn't have encoding and content - readmeContentsResponse.Encoding = nil - readmeContentsResponse.Content = nil - - expectedContentsListResponse := []*api.ContentsResponse{ - readmeContentsResponse, - } - - t.Run("Get root dir contents with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, ref) - assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get root dir contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, "") - assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) - assert.NoError(t, err) - }) -} - -func TestGetContentsOrListForFile(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - treePath := "README.md" - ref := ctx.Repo.Repository.DefaultBranch - - expectedContentsResponse := getExpectedReadmeContentsResponse() - - t.Run("Get README.md contents with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, ref) - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, "") - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) -} - -func TestGetContentsErrors(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - treePath := "README.md" - ref := repo.DefaultBranch - - t.Run("bad treePath", func(t *testing.T) { - badTreePath := "bad/tree.md" - fileContentResponse, err := GetContents(ctx, repo, badTreePath, ref, false) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: , rel_path: bad]") - assert.Nil(t, fileContentResponse) - }) - - t.Run("bad ref", func(t *testing.T) { - badRef := "bad_ref" - fileContentResponse, err := GetContents(ctx, repo, treePath, badRef, false) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") - assert.Nil(t, fileContentResponse) - }) -} - -func TestGetContentsOrListErrors(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - repo := ctx.Repo.Repository - treePath := "README.md" - ref := repo.DefaultBranch - - t.Run("bad treePath", func(t *testing.T) { - badTreePath := "bad/tree.md" - fileContentResponse, err := GetContentsOrList(ctx, repo, badTreePath, ref) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: , rel_path: bad]") - assert.Nil(t, fileContentResponse) - }) - - t.Run("bad ref", func(t *testing.T) { - badRef := "bad_ref" - fileContentResponse, err := GetContentsOrList(ctx, repo, treePath, badRef) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") - assert.Nil(t, fileContentResponse) - }) -} - -func TestGetContentsOrListOfEmptyRepos(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user30/empty") - ctx.SetPathParam("id", "52") - contexttest.LoadRepo(t, ctx, 52) - contexttest.LoadUser(t, ctx, 30) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - - t.Run("empty repo", func(t *testing.T) { - contents, err := GetContentsOrList(ctx, repo, "", "") + // GetContentsOrList's behavior is fully tested in integration tests, so we don't need to test it here. + + t.Run("GetBlobBySHA", func(t *testing.T) { + sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" + ctx.SetPathParam("id", "1") + ctx.SetPathParam("sha", sha) + gbr, err := GetBlobBySHA(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("sha")) + expectedGBR := &api.GitBlobResponse{ + Content: util.ToPointer("dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK"), + Encoding: util.ToPointer("base64"), + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + Size: 180, + } assert.NoError(t, err) - assert.Empty(t, contents) + assert.Equal(t, expectedGBR, gbr) }) } - -func TestGetBlobBySHA(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d" - ctx.SetPathParam("id", "1") - ctx.SetPathParam("sha", sha) - - gitRepo, err := gitrepo.OpenRepository(ctx, ctx.Repo.Repository) - if err != nil { - t.Fail() - } - - gbr, err := GetBlobBySHA(ctx, ctx.Repo.Repository, gitRepo, ctx.PathParam("sha")) - expectedGBR := &api.GitBlobResponse{ - Content: "dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK", - Encoding: "base64", - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", - SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - Size: 180, - } - assert.NoError(t, err) - assert.Equal(t, expectedGBR, gbr) -} diff --git a/services/repository/files/diff.go b/services/repository/files/diff.go index 0b3550452a..50d01f9d7c 100644 --- a/services/repository/files/diff.go +++ b/services/repository/files/diff.go @@ -29,7 +29,7 @@ func GetDiffPreview(ctx context.Context, repo *repo_model.Repository, branch, tr } // Add the object to the database - objectHash, err := t.HashObject(ctx, strings.NewReader(content)) + objectHash, err := t.HashObjectAndWrite(ctx, strings.NewReader(content)) if err != nil { return nil, err } diff --git a/services/repository/files/diff_test.go b/services/repository/files/diff_test.go index a8514791cc..ae702e4189 100644 --- a/services/repository/files/diff_test.go +++ b/services/repository/files/diff_test.go @@ -118,7 +118,7 @@ func TestGetDiffPreview(t *testing.T) { assert.NoError(t, err) bs, err := json.Marshal(diff) assert.NoError(t, err) - assert.EqualValues(t, string(expectedBs), string(bs)) + assert.Equal(t, string(expectedBs), string(bs)) }) t.Run("empty branch, same results", func(t *testing.T) { @@ -128,7 +128,7 @@ func TestGetDiffPreview(t *testing.T) { assert.NoError(t, err) bs, err := json.Marshal(diff) assert.NoError(t, err) - assert.EqualValues(t, expectedBs, bs) + assert.Equal(t, expectedBs, bs) }) } diff --git a/services/repository/files/file.go b/services/repository/files/file.go index 2caa1b4946..f48e32b427 100644 --- a/services/repository/files/file.go +++ b/services/repository/files/file.go @@ -5,6 +5,7 @@ package files import ( "context" + "errors" "fmt" "net/url" "strings" @@ -12,18 +13,40 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" ) -func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch string, treeNames []string) (*api.FilesResponse, error) { - files := []*api.ContentsResponse{} - for _, file := range treeNames { - fileContents, _ := GetContents(ctx, repo, file, branch, false) // ok if fails, then will be nil +func GetContentsListFromTreePaths(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, treePaths []string) (files []*api.ContentsResponse) { + var size int64 + for _, treePath := range treePaths { + // ok if fails, then will be nil + fileContents, _ := GetFileContents(ctx, repo, gitRepo, refCommit, GetContentsOrListOptions{ + TreePath: treePath, + IncludeSingleFileContent: true, + IncludeCommitMetadata: true, + }) + if fileContents != nil && fileContents.Content != nil && *fileContents.Content != "" { + // if content isn't empty (e.g., due to the single blob being too large), add file size to response size + size += int64(len(*fileContents.Content)) + } + if size > setting.API.DefaultMaxResponseSize { + break // stop if max response size would be exceeded + } files = append(files, fileContents) + if len(files) == setting.API.DefaultPagingNum { + break // stop if paging num reached + } } - fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil - verification := GetPayloadCommitVerification(ctx, commit) + return files +} + +func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, treeNames []string) (*api.FilesResponse, error) { + files := GetContentsListFromTreePaths(ctx, repo, gitRepo, refCommit, treeNames) + fileCommitResponse, _ := GetFileCommitResponse(repo, refCommit.Commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, refCommit.Commit) filesResponse := &api.FilesResponse{ Files: files, Commit: fileCommitResponse, @@ -32,19 +55,6 @@ func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository return filesResponse, nil } -// GetFileResponseFromCommit Constructs a FileResponse from a Commit object -func GetFileResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch, treeName string) (*api.FileResponse, error) { - fileContents, _ := GetContents(ctx, repo, treeName, branch, false) // ok if fails, then will be nil - fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil - verification := GetPayloadCommitVerification(ctx, commit) - fileResponse := &api.FileResponse{ - Content: fileContents, - Commit: fileCommitResponse, - Verification: verification, - } - return fileResponse, nil -} - // constructs a FileResponse with the file at the index from FilesResponse func GetFileResponseFromFilesResponse(filesResponse *api.FilesResponse, index int) *api.FileResponse { content := &api.ContentsResponse{} @@ -62,10 +72,10 @@ func GetFileResponseFromFilesResponse(filesResponse *api.FilesResponse, index in // GetFileCommitResponse Constructs a FileCommitResponse from a Commit object func GetFileCommitResponse(repo *repo_model.Repository, commit *git.Commit) (*api.FileCommitResponse, error) { if repo == nil { - return nil, fmt.Errorf("repo cannot be nil") + return nil, errors.New("repo cannot be nil") } if commit == nil { - return nil, fmt.Errorf("commit cannot be nil") + return nil, errors.New("commit cannot be nil") } commitURL, _ := url.Parse(repo.APIURL() + "/git/commits/" + url.PathEscape(commit.ID.String())) commitTreeURL, _ := url.Parse(repo.APIURL() + "/git/trees/" + url.PathEscape(commit.Tree.ID.String())) @@ -129,15 +139,17 @@ func (err ErrFilenameInvalid) Unwrap() error { return util.ErrInvalidArgument } -// CleanUploadFileName Trims a filename and returns empty string if it is a .git directory -func CleanUploadFileName(name string) string { - // Rebase the filename +// CleanGitTreePath cleans a tree path for git, it returns an empty string the path is invalid (e.g.: contains ".git" part) +func CleanGitTreePath(name string) string { name = util.PathJoinRel(name) // Git disallows any filenames to have a .git directory in them. - for _, part := range strings.Split(name, "/") { - if strings.ToLower(part) == ".git" { + for part := range strings.SplitSeq(name, "/") { + if strings.EqualFold(part, ".git") { return "" } } + if name == "." { + name = "" + } return name } diff --git a/services/repository/files/file_test.go b/services/repository/files/file_test.go index 52c0574883..cdb6a266ff 100644 --- a/services/repository/files/file_test.go +++ b/services/repository/files/file_test.go @@ -6,115 +6,22 @@ package files import ( "testing" - "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/gitrepo" - "code.gitea.io/gitea/modules/setting" - api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/services/contexttest" - "github.com/stretchr/testify/assert" ) func TestCleanUploadFileName(t *testing.T) { - t.Run("Clean regular file", func(t *testing.T) { - name := "this/is/test" - cleanName := CleanUploadFileName(name) - expectedCleanName := name - assert.EqualValues(t, expectedCleanName, cleanName) - }) - - t.Run("Clean a .git path", func(t *testing.T) { - name := "this/is/test/.git" - cleanName := CleanUploadFileName(name) - expectedCleanName := "" - assert.EqualValues(t, expectedCleanName, cleanName) - }) -} - -func getExpectedFileResponse() *api.FileResponse { - treePath := "README.md" - sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" - encoding := "base64" - content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" - selfURL := setting.AppURL + "api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" - htmlURL := setting.AppURL + "user2/repo1/src/branch/master/" + treePath - gitURL := setting.AppURL + "api/v1/repos/user2/repo1/git/blobs/" + sha - downloadURL := setting.AppURL + "user2/repo1/raw/branch/master/" + treePath - return &api.FileResponse{ - Content: &api.ContentsResponse{ - Name: treePath, - Path: treePath, - SHA: sha, - LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - Type: "file", - Size: 30, - Encoding: &encoding, - Content: &content, - URL: &selfURL, - HTMLURL: &htmlURL, - GitURL: &gitURL, - DownloadURL: &downloadURL, - Links: &api.FileLinksResponse{ - Self: &selfURL, - GitURL: &gitURL, - HTMLURL: &htmlURL, - }, - }, - Commit: &api.FileCommitResponse{ - CommitMeta: api.CommitMeta{ - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/65f1bf27bc3bf70f64657658635e66094edbcb4d", - SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - }, - HTMLURL: "https://try.gitea.io/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d", - Author: &api.CommitUser{ - Identity: api.Identity{ - Name: "user1", - Email: "address1@example.com", - }, - Date: "2017-03-19T20:47:59Z", - }, - Committer: &api.CommitUser{ - Identity: api.Identity{ - Name: "Ethan Koenig", - Email: "ethantkoenig@gmail.com", - }, - Date: "2017-03-19T20:47:59Z", - }, - Parents: []*api.CommitMeta{}, - Message: "Initial commit\n", - Tree: &api.CommitMeta{ - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/2a2f1d4670728a2e10049e345bd7a276468beab6", - SHA: "2a2f1d4670728a2e10049e345bd7a276468beab6", - }, - }, - Verification: &api.PayloadCommitVerification{ - Verified: false, - Reason: "gpg.error.not_signed_commit", - Signature: "", - Payload: "", - }, + cases := []struct { + input, expected string + }{ + {"", ""}, + {".", ""}, + {"a/./b", "a/b"}, + {"a.git", "a.git"}, + {".git/b", ""}, + {"a/.git", ""}, + {"/a/../../b", "b"}, + } + for _, c := range cases { + assert.Equal(t, c.expected, CleanGitTreePath(c.input), "input: %q", c.input) } -} - -func TestGetFileResponseFromCommit(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - branch := repo.DefaultBranch - treePath := "README.md" - gitRepo, _ := gitrepo.OpenRepository(ctx, repo) - defer gitRepo.Close() - commit, _ := gitRepo.GetBranchCommit(branch) - expectedFileResponse := getExpectedFileResponse() - - fileResponse, err := GetFileResponseFromCommit(ctx, repo, commit, branch, treePath) - assert.NoError(t, err) - assert.EqualValues(t, expectedFileResponse, fileResponse) } diff --git a/services/repository/files/patch.go b/services/repository/files/patch.go index 1941adb86a..11a8744b7f 100644 --- a/services/repository/files/patch.go +++ b/services/repository/files/patch.go @@ -12,7 +12,6 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -45,7 +44,6 @@ type ApplyDiffPatchOptions struct { NewBranch string Message string Content string - SHA string Author *IdentityOptions Committer *IdentityOptions Dates *CommitDateOptions @@ -62,29 +60,26 @@ func (opts *ApplyDiffPatchOptions) Validate(ctx context.Context, repo *repo_mode opts.NewBranch = opts.OldBranch } - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return err - } - defer closer.Close() - // oldBranch must exist for this operation - if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil { + if exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.OldBranch); err != nil { return err + } else if !exist { + return git_model.ErrBranchNotExist{ + BranchName: opts.OldBranch, + } } // A NewBranch can be specified for the patch to be applied to. // Check to make sure the branch does not already exist, otherwise we can't proceed. // If we aren't branching to a new branch, make sure user can commit to the given branch if opts.NewBranch != opts.OldBranch { - existingBranch, err := gitRepo.GetBranch(opts.NewBranch) - if existingBranch != nil { + exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.NewBranch) + if err != nil { + return err + } else if exist { return git_model.ErrBranchAlreadyExists{ BranchName: opts.NewBranch, } } - if err != nil && !git.IsErrBranchNotExist(err) { - return err - } } else { protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, opts.OldBranch) if err != nil { diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go index d2c70a7a34..c2f61c8223 100644 --- a/services/repository/files/temp_repo.go +++ b/services/repository/files/temp_repo.go @@ -6,6 +6,7 @@ package files import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -29,23 +30,24 @@ type TemporaryUploadRepository struct { repo *repo_model.Repository gitRepo *git.Repository basePath string + cleanup func() } // NewTemporaryUploadRepository creates a new temporary upload repository func NewTemporaryUploadRepository(repo *repo_model.Repository) (*TemporaryUploadRepository, error) { - basePath, err := repo_module.CreateTemporaryPath("upload") + basePath, cleanup, err := repo_module.CreateTemporaryPath("upload") if err != nil { return nil, err } - t := &TemporaryUploadRepository{repo: repo, basePath: basePath} + t := &TemporaryUploadRepository{repo: repo, basePath: basePath, cleanup: cleanup} return t, nil } // Close the repository cleaning up all files func (t *TemporaryUploadRepository) Close() { defer t.gitRepo.Close() - if err := repo_module.RemoveTemporaryPath(t.basePath); err != nil { - log.Error("Failed to remove temporary path %s: %v", t.basePath, err) + if t.cleanup != nil { + t.cleanup() } } @@ -126,7 +128,7 @@ func (t *TemporaryUploadRepository) LsFiles(ctx context.Context, filenames ...st } fileList := make([]string, 0, len(filenames)) - for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) { + for line := range bytes.SplitSeq(stdOut.Bytes(), []byte{'\000'}) { fileList = append(fileList, string(line)) } @@ -162,8 +164,8 @@ func (t *TemporaryUploadRepository) RemoveFilesFromIndex(ctx context.Context, fi return nil } -// HashObject writes the provided content to the object db and returns its hash -func (t *TemporaryUploadRepository) HashObject(ctx context.Context, content io.Reader) (string, error) { +// HashObjectAndWrite writes the provided content to the object db and returns its hash +func (t *TemporaryUploadRepository) HashObjectAndWrite(ctx context.Context, content io.Reader) (string, error) { stdOut := new(bytes.Buffer) stdErr := new(bytes.Buffer) @@ -291,15 +293,18 @@ func (t *TemporaryUploadRepository) CommitTree(ctx context.Context, opts *Commit } var sign bool - var keyID string + var key *git.SigningKey var signer *git.Signature if opts.ParentCommitID != "" { - sign, keyID, signer, _ = asymkey_service.SignCRUDAction(ctx, t.repo.RepoPath(), opts.DoerUser, t.basePath, opts.ParentCommitID) + sign, key, signer, _ = asymkey_service.SignCRUDAction(ctx, t.repo.RepoPath(), opts.DoerUser, t.basePath, opts.ParentCommitID) } else { - sign, keyID, signer, _ = asymkey_service.SignInitialCommit(ctx, t.repo.RepoPath(), opts.DoerUser) + sign, key, signer, _ = asymkey_service.SignInitialCommit(ctx, t.repo.RepoPath(), opts.DoerUser) } if sign { - cmdCommitTree.AddOptionFormat("-S%s", keyID) + if key.Format != "" { + cmdCommitTree.AddConfig("gpg.format", key.Format) + } + cmdCommitTree.AddOptionFormat("-S%s", key.KeyID) if t.repo.GetTrustModel() == repo_model.CommitterTrustModel || t.repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { if committerSig.Name != authorSig.Name || committerSig.Email != authorSig.Email { // Add trailers @@ -414,7 +419,7 @@ func (t *TemporaryUploadRepository) DiffIndex(ctx context.Context) (*gitdiff.Dif // GetBranchCommit Gets the commit object of the given branch func (t *TemporaryUploadRepository) GetBranchCommit(branch string) (*git.Commit, error) { if t.gitRepo == nil { - return nil, fmt.Errorf("repository has not been cloned") + return nil, errors.New("repository has not been cloned") } return t.gitRepo.GetBranchCommit(branch) } @@ -422,7 +427,7 @@ func (t *TemporaryUploadRepository) GetBranchCommit(branch string) (*git.Commit, // GetCommit Gets the commit object of the given commit ID func (t *TemporaryUploadRepository) GetCommit(commitID string) (*git.Commit, error) { if t.gitRepo == nil { - return nil, fmt.Errorf("repository has not been cloned") + return nil, errors.New("repository has not been cloned") } return t.gitRepo.GetCommit(commitID) } diff --git a/services/repository/files/tree.go b/services/repository/files/tree.go index 9142416347..e481a3e7d2 100644 --- a/services/repository/files/tree.go +++ b/services/repository/files/tree.go @@ -6,12 +6,14 @@ package files import ( "context" "fmt" + "html/template" "net/url" "path" "sort" "strings" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/fileicon" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -88,15 +90,8 @@ func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git if rangeStart >= len(entries) { return tree, nil } - var rangeEnd int - if len(entries) > perPage { - tree.Truncated = true - } - if rangeStart+perPage < len(entries) { - rangeEnd = rangeStart + perPage - } else { - rangeEnd = len(entries) - } + rangeEnd := min(rangeStart+perPage, len(entries)) + tree.Truncated = rangeEnd < len(entries) tree.Entries = make([]api.GitEntry, rangeEnd-rangeStart) for e := rangeStart; e < rangeEnd; e++ { i := e - rangeStart @@ -140,8 +135,13 @@ func entryModeString(entryMode git.EntryMode) string { } type TreeViewNode struct { - EntryName string `json:"entryName"` - EntryMode string `json:"entryMode"` + EntryName string `json:"entryName"` + EntryMode string `json:"entryMode"` + EntryIcon template.HTML `json:"entryIcon"` + EntryIconOpen template.HTML `json:"entryIconOpen,omitempty"` + + SymLinkedToMode string `json:"symLinkedToMode,omitempty"` // TODO: for the EntryMode="symlink" + FullPath string `json:"fullPath"` SubmoduleURL string `json:"submoduleUrl,omitempty"` Children []*TreeViewNode `json:"children,omitempty"` @@ -151,20 +151,29 @@ func (node *TreeViewNode) sortLevel() int { return util.Iif(node.EntryMode == "tree" || node.EntryMode == "commit", 0, 1) } -func newTreeViewNodeFromEntry(ctx context.Context, commit *git.Commit, parentDir string, entry *git.TreeEntry) *TreeViewNode { +func newTreeViewNodeFromEntry(ctx context.Context, repoLink string, renderedIconPool *fileicon.RenderedIconPool, commit *git.Commit, parentDir string, entry *git.TreeEntry) *TreeViewNode { node := &TreeViewNode{ EntryName: entry.Name(), EntryMode: entryModeString(entry.Mode()), FullPath: path.Join(parentDir, entry.Name()), } + entryInfo := fileicon.EntryInfoFromGitTreeEntry(commit, node.FullPath, entry) + node.EntryIcon = fileicon.RenderEntryIconHTML(renderedIconPool, entryInfo) + if entryInfo.EntryMode.IsDir() { + entryInfo.IsOpen = true + node.EntryIconOpen = fileicon.RenderEntryIconHTML(renderedIconPool, entryInfo) + } + if node.EntryMode == "commit" { if subModule, err := commit.GetSubModule(node.FullPath); err != nil { log.Error("GetSubModule: %v", err) } else if subModule != nil { - submoduleFile := git.NewCommitSubmoduleFile(subModule.URL, entry.ID.String()) - webLink := submoduleFile.SubmoduleWebLink(ctx) - node.SubmoduleURL = webLink.CommitWebLink + submoduleFile := git.NewCommitSubmoduleFile(repoLink, node.FullPath, subModule.URL, entry.ID.String()) + webLink := submoduleFile.SubmoduleWebLinkTree(ctx) + if webLink != nil { + node.SubmoduleURL = webLink.CommitWebLink + } } } @@ -182,7 +191,7 @@ func sortTreeViewNodes(nodes []*TreeViewNode) { }) } -func listTreeNodes(ctx context.Context, commit *git.Commit, tree *git.Tree, treePath, subPath string) ([]*TreeViewNode, error) { +func listTreeNodes(ctx context.Context, repoLink string, renderedIconPool *fileicon.RenderedIconPool, commit *git.Commit, tree *git.Tree, treePath, subPath string) ([]*TreeViewNode, error) { entries, err := tree.ListEntries() if err != nil { return nil, err @@ -191,14 +200,14 @@ func listTreeNodes(ctx context.Context, commit *git.Commit, tree *git.Tree, tree subPathDirName, subPathRemaining, _ := strings.Cut(subPath, "/") nodes := make([]*TreeViewNode, 0, len(entries)) for _, entry := range entries { - node := newTreeViewNodeFromEntry(ctx, commit, treePath, entry) + node := newTreeViewNodeFromEntry(ctx, repoLink, renderedIconPool, commit, treePath, entry) nodes = append(nodes, node) if entry.IsDir() && subPathDirName == entry.Name() { subTreePath := treePath + "/" + node.EntryName if subTreePath[0] == '/' { subTreePath = subTreePath[1:] } - subNodes, err := listTreeNodes(ctx, commit, entry.Tree(), subTreePath, subPathRemaining) + subNodes, err := listTreeNodes(ctx, repoLink, renderedIconPool, commit, entry.Tree(), subTreePath, subPathRemaining) if err != nil { log.Error("listTreeNodes: %v", err) } else { @@ -210,10 +219,10 @@ func listTreeNodes(ctx context.Context, commit *git.Commit, tree *git.Tree, tree return nodes, nil } -func GetTreeViewNodes(ctx context.Context, commit *git.Commit, treePath, subPath string) ([]*TreeViewNode, error) { +func GetTreeViewNodes(ctx context.Context, repoLink string, renderedIconPool *fileicon.RenderedIconPool, commit *git.Commit, treePath, subPath string) ([]*TreeViewNode, error) { entry, err := commit.GetTreeEntryByPath(treePath) if err != nil { return nil, err } - return listTreeNodes(ctx, commit, entry.Tree(), treePath, subPath) + return listTreeNodes(ctx, repoLink, renderedIconPool, commit, entry.Tree(), treePath, subPath) } diff --git a/services/repository/files/tree_test.go b/services/repository/files/tree_test.go index 8ea54969ce..38ac9f25fc 100644 --- a/services/repository/files/tree_test.go +++ b/services/repository/files/tree_test.go @@ -4,9 +4,11 @@ package files import ( + "html/template" "testing" "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/fileicon" "code.gitea.io/gitea/modules/git" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/services/contexttest" @@ -49,7 +51,7 @@ func TestGetTreeBySHA(t *testing.T) { TotalCount: 1, } - assert.EqualValues(t, expectedTree, tree) + assert.Equal(t, expectedTree, tree) } func TestGetTreeViewNodes(t *testing.T) { @@ -62,40 +64,57 @@ func TestGetTreeViewNodes(t *testing.T) { contexttest.LoadGitRepo(t, ctx) defer ctx.Repo.GitRepo.Close() - treeNodes, err := GetTreeViewNodes(ctx, ctx.Repo.Commit, "", "") + curRepoLink := "/any/repo-link" + renderedIconPool := fileicon.NewRenderedIconPool() + mockIconForFile := func(id string) template.HTML { + return template.HTML(`<svg class="svg git-entry-icon octicon-file" width="16" height="16" aria-hidden="true"><use xlink:href="#` + id + `"></use></svg>`) + } + mockIconForFolder := func(id string) template.HTML { + return template.HTML(`<svg class="svg git-entry-icon octicon-file-directory-fill" width="16" height="16" aria-hidden="true"><use xlink:href="#` + id + `"></use></svg>`) + } + mockOpenIconForFolder := func(id string) template.HTML { + return template.HTML(`<svg class="svg git-entry-icon octicon-file-directory-open-fill" width="16" height="16" aria-hidden="true"><use xlink:href="#` + id + `"></use></svg>`) + } + treeNodes, err := GetTreeViewNodes(ctx, curRepoLink, renderedIconPool, ctx.Repo.Commit, "", "") assert.NoError(t, err) assert.Equal(t, []*TreeViewNode{ { - EntryName: "docs", - EntryMode: "tree", - FullPath: "docs", + EntryName: "docs", + EntryMode: "tree", + FullPath: "docs", + EntryIcon: mockIconForFolder(`svg-mfi-folder-docs`), + EntryIconOpen: mockOpenIconForFolder(`svg-mfi-folder-docs`), }, }, treeNodes) - treeNodes, err = GetTreeViewNodes(ctx, ctx.Repo.Commit, "", "docs/README.md") + treeNodes, err = GetTreeViewNodes(ctx, curRepoLink, renderedIconPool, ctx.Repo.Commit, "", "docs/README.md") assert.NoError(t, err) assert.Equal(t, []*TreeViewNode{ { - EntryName: "docs", - EntryMode: "tree", - FullPath: "docs", + EntryName: "docs", + EntryMode: "tree", + FullPath: "docs", + EntryIcon: mockIconForFolder(`svg-mfi-folder-docs`), + EntryIconOpen: mockOpenIconForFolder(`svg-mfi-folder-docs`), Children: []*TreeViewNode{ { EntryName: "README.md", EntryMode: "blob", FullPath: "docs/README.md", + EntryIcon: mockIconForFile(`svg-mfi-readme`), }, }, }, }, treeNodes) - treeNodes, err = GetTreeViewNodes(ctx, ctx.Repo.Commit, "docs", "README.md") + treeNodes, err = GetTreeViewNodes(ctx, curRepoLink, renderedIconPool, ctx.Repo.Commit, "docs", "README.md") assert.NoError(t, err) assert.Equal(t, []*TreeViewNode{ { EntryName: "README.md", EntryMode: "blob", FullPath: "docs/README.md", + EntryIcon: mockIconForFile(`svg-mfi-readme`), }, }, treeNodes) } diff --git a/services/repository/files/update.go b/services/repository/files/update.go index cade7ba2bf..e871f777e5 100644 --- a/services/repository/files/update.go +++ b/services/repository/files/update.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "path" + "slices" "strings" "time" @@ -15,12 +16,14 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" asymkey_service "code.gitea.io/gitea/services/asymkey" pull_service "code.gitea.io/gitea/services/pull" ) @@ -85,14 +88,32 @@ func (err ErrRepoFileDoesNotExist) Unwrap() error { return util.ErrNotExist } +type LazyReadSeeker interface { + io.ReadSeeker + io.Closer + OpenLazyReader() error +} + // ChangeRepoFiles adds, updates or removes multiple files in the given repository -func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ChangeRepoFilesOptions) (*structs.FilesResponse, error) { +func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ChangeRepoFilesOptions) (_ *structs.FilesResponse, errRet error) { + var addedLfsPointers []lfs.Pointer + defer func() { + if errRet != nil { + for _, lfsPointer := range addedLfsPointers { + _, err := git_model.RemoveLFSMetaObjectByOid(ctx, repo.ID, lfsPointer.Oid) + if err != nil { + log.Error("ChangeRepoFiles: RemoveLFSMetaObjectByOid failed: %v", err) + } + } + } + }() + err := repo.MustNotBeArchived() if err != nil { return nil, err } - // If no branch name is set, assume default branch + // If no branch name is set, assume the default branch if opts.OldBranch == "" { opts.OldBranch = repo.DefaultBranch } @@ -107,8 +128,13 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use defer closer.Close() // oldBranch must exist for this operation - if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil && !repo.IsEmpty { + if exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.OldBranch); err != nil { return nil, err + } else if !exist && !repo.IsEmpty { + return nil, git_model.ErrBranchNotExist{ + RepoID: repo.ID, + BranchName: opts.OldBranch, + } } var treePaths []string @@ -119,14 +145,14 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } // Check that the path given in opts.treePath is valid (not a git path) - treePath := CleanUploadFileName(file.TreePath) + treePath := CleanGitTreePath(file.TreePath) if treePath == "" { return nil, ErrFilenameInvalid{ Path: file.TreePath, } } // If there is a fromTreePath (we are copying it), also clean it up - fromTreePath := CleanUploadFileName(file.FromTreePath) + fromTreePath := CleanGitTreePath(file.FromTreePath) if fromTreePath == "" && file.FromTreePath != "" { return nil, ErrFilenameInvalid{ Path: file.FromTreePath, @@ -145,15 +171,15 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use // Check to make sure the branch does not already exist, otherwise we can't proceed. // If we aren't branching to a new branch, make sure user can commit to the given branch if opts.NewBranch != opts.OldBranch { - existingBranch, err := gitRepo.GetBranch(opts.NewBranch) - if existingBranch != nil { + exist, err := git_model.IsBranchExist(ctx, repo.ID, opts.NewBranch) + if err != nil { + return nil, err + } + if exist { return nil, git_model.ErrBranchAlreadyExists{ BranchName: opts.NewBranch, } } - if err != nil && !git.IsErrBranchNotExist(err) { - return nil, err - } } else if err := VerifyBranchProtection(ctx, repo, doer, opts.OldBranch, treePaths); err != nil { return nil, err } @@ -196,13 +222,7 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } // Find the file we want to delete in the index - inFilelist := false - for _, indexFile := range filesInIndex { - if indexFile == file.TreePath { - inFilelist = true - break - } - } + inFilelist := slices.Contains(filesInIndex, file.TreePath) if !inFilelist { return nil, ErrRepoFileDoesNotExist{ Path: file.TreePath, @@ -218,7 +238,7 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use return nil, err // Couldn't get a commit for the branch } - // Assigned LastCommitID in opts if it hasn't been set + // Assigned LastCommitID in "opts" if it hasn't been set if opts.LastCommitID == "" { opts.LastCommitID = commit.ID.String() } else { @@ -230,22 +250,25 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } for _, file := range opts.Files { - if err := handleCheckErrors(file, commit, opts); err != nil { + if err = handleCheckErrors(file, commit, opts); err != nil { return nil, err } } } - contentStore := lfs.NewContentStore() + lfsContentStore := lfs.NewContentStore() for _, file := range opts.Files { switch file.Operation { - case "create", "update": - if err := CreateOrUpdateFile(ctx, t, file, contentStore, repo.ID, hasOldBranch); err != nil { + case "create", "update", "rename", "upload": + addedLfsPointer, err := modifyFile(ctx, t, file, lfsContentStore, repo.ID) + if err != nil { return nil, err } + if addedLfsPointer != nil { + addedLfsPointers = append(addedLfsPointers, *addedLfsPointer) + } case "delete": - // Remove the file from the index - if err := t.RemoveFilesFromIndex(ctx, file.TreePath); err != nil { + if err = t.RemoveFilesFromIndex(ctx, file.TreePath); err != nil { return nil, err } default: @@ -290,14 +313,16 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use return nil, err } - filesResponse, err := GetFilesResponseFromCommit(ctx, repo, commit, opts.NewBranch, treePaths) + // FIXME: this call seems not right, why it needs to read the file content again + // FIXME: why it uses the NewBranch as "ref", it should use the commit ID because the response is only for this commit + filesResponse, err := GetFilesResponseFromCommit(ctx, repo, gitRepo, utils.NewRefCommit(git.RefNameFromBranch(opts.NewBranch), commit), treePaths) if err != nil { return nil, err } if repo.IsEmpty { if isEmpty, err := gitRepo.IsEmpty(); err == nil && !isEmpty { - _ = repo_model.UpdateRepositoryCols(ctx, &repo_model.Repository{ID: repo.ID, IsEmpty: false, DefaultBranch: opts.NewBranch}, "is_empty", "default_branch") + _ = repo_model.UpdateRepositoryColsWithAutoTime(ctx, &repo_model.Repository{ID: repo.ID, IsEmpty: false, DefaultBranch: opts.NewBranch}, "is_empty", "default_branch") } } @@ -363,22 +388,33 @@ func (err ErrSHAOrCommitIDNotProvided) Error() string { // handles the check for various issues for ChangeRepoFiles func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRepoFilesOptions) error { - if file.Operation == "update" || file.Operation == "delete" { - fromEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) - if err != nil { - return err + // check old entry (fromTreePath/fromEntry) + if file.Operation == "update" || file.Operation == "upload" || file.Operation == "delete" || file.Operation == "rename" { + var fromEntryIDString string + { + fromEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) + if file.Operation == "upload" && git.IsErrNotExist(err) { + fromEntry = nil + } else if err != nil { + return err + } + if fromEntry != nil { + fromEntryIDString = fromEntry.ID.String() + file.Options.executable = fromEntry.IsExecutable() // FIXME: legacy hacky approach, it shouldn't prepare the "Options" in the "check" function + } } + if file.SHA != "" { - // If a SHA was given and the SHA given doesn't match the SHA of the fromTreePath, throw error - if file.SHA != fromEntry.ID.String() { + // If the SHA given doesn't match the SHA of the fromTreePath, throw error + if file.SHA != fromEntryIDString { return pull_service.ErrSHADoesNotMatch{ Path: file.Options.treePath, GivenSHA: file.SHA, - CurrentSHA: fromEntry.ID.String(), + CurrentSHA: fromEntryIDString, } } } else if opts.LastCommitID != "" { - // If a lastCommitID was given and it doesn't match the commitID of the head of the branch throw + // If a lastCommitID given doesn't match the branch head's commitID throw // an error, but only if we aren't creating a new branch. if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch { if changed, err := commit.FileChangedSinceCommit(file.Options.treePath, opts.LastCommitID); err != nil { @@ -396,13 +432,13 @@ func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRep // haven't been made. We throw an error if one wasn't provided. return ErrSHAOrCommitIDNotProvided{} } - file.Options.executable = fromEntry.IsExecutable() } - if file.Operation == "create" || file.Operation == "update" { - // For the path where this file will be created/updated, we need to make - // sure no parts of the path are existing files or links except for the last - // item in the path which is the file name, and that shouldn't exist IF it is - // a new file OR is being moved to a new path. + + // check new entry (treePath/treeEntry) + if file.Operation == "create" || file.Operation == "update" || file.Operation == "upload" || file.Operation == "rename" { + // For operation's target path, we need to make sure no parts of the path are existing files or links + // except for the last item in the path (which is the file name). + // And that shouldn't exist IF it is a new file OR is being moved to a new path. treePathParts := strings.Split(file.Options.treePath, "/") subTreePath := "" for index, part := range treePathParts { @@ -439,7 +475,7 @@ func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRep Type: git.EntryModeTree, } } else if file.Options.fromTreePath != file.Options.treePath || file.Operation == "create" { - // The entry shouldn't exist if we are creating new file or moving to a new path + // The entry shouldn't exist if we are creating the new file or moving to a new path return ErrRepoFileAlreadyExists{ Path: file.Options.treePath, } @@ -450,21 +486,23 @@ func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRep return nil } -// CreateOrUpdateFile handles creating or updating a file for ChangeRepoFiles -func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile, contentStore *lfs.ContentStore, repoID int64, hasOldBranch bool) error { +func modifyFile(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile, contentStore *lfs.ContentStore, repoID int64) (addedLfsPointer *lfs.Pointer, _ error) { + if rd, ok := file.ContentReader.(LazyReadSeeker); ok { + if err := rd.OpenLazyReader(); err != nil { + return nil, fmt.Errorf("OpenLazyReader: %w", err) + } + defer rd.Close() + } + // Get the two paths (might be the same if not moving) from the index if they exist filesInIndex, err := t.LsFiles(ctx, file.TreePath, file.FromTreePath) if err != nil { - return fmt.Errorf("UpdateRepoFile: %w", err) + return nil, fmt.Errorf("LsFiles: %w", err) } // If is a new file (not updating) then the given path shouldn't exist if file.Operation == "create" { - for _, indexFile := range filesInIndex { - if indexFile == file.TreePath { - return ErrRepoFileAlreadyExists{ - Path: file.TreePath, - } - } + if slices.Contains(filesInIndex, file.TreePath) { + return nil, ErrRepoFileAlreadyExists{Path: file.TreePath} } } @@ -472,79 +510,178 @@ func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file if file.Options.fromTreePath != file.Options.treePath && len(filesInIndex) > 0 { for _, indexFile := range filesInIndex { if indexFile == file.Options.fromTreePath { - if err := t.RemoveFilesFromIndex(ctx, file.FromTreePath); err != nil { - return err + if err = t.RemoveFilesFromIndex(ctx, file.FromTreePath); err != nil { + return nil, err } } } } - treeObjectContentReader := file.ContentReader - var lfsMetaObject *git_model.LFSMetaObject - if setting.LFS.StartServer && hasOldBranch { - // Check there is no way this can return multiple infos - filename2attribute2info, err := t.gitRepo.CheckAttribute(git.CheckAttributeOpts{ - Attributes: []string{"filter"}, - Filenames: []string{file.Options.treePath}, - CachedOnly: true, - }) + var writeObjectRet *writeRepoObjectRet + switch file.Operation { + case "create", "update", "upload": + writeObjectRet, err = writeRepoObjectForModify(ctx, t, file) + case "rename": + writeObjectRet, err = writeRepoObjectForRename(ctx, t, file) + default: + return nil, util.NewInvalidArgumentErrorf("unknown file modification operation: '%s'", file.Operation) + } + if err != nil { + return nil, err + } + + // Add the object to the index, the "file.Options.executable" is set in handleCheckErrors by the caller (legacy hacky approach) + if err = t.AddObjectToIndex(ctx, util.Iif(file.Options.executable, "100755", "100644"), writeObjectRet.ObjectHash, file.Options.treePath); err != nil { + return nil, err + } + + if writeObjectRet.LfsContent == nil { + return nil, nil // No LFS pointer, so nothing to do + } + defer writeObjectRet.LfsContent.Close() + + // Now we must store the content into an LFS object + lfsMetaObject, err := git_model.NewLFSMetaObject(ctx, repoID, writeObjectRet.LfsPointer) + if err != nil { + return nil, err + } + exist, err := contentStore.Exists(lfsMetaObject.Pointer) + if err != nil { + return nil, err + } + if !exist { + err = contentStore.Put(lfsMetaObject.Pointer, writeObjectRet.LfsContent) if err != nil { - return err + if _, errRemove := git_model.RemoveLFSMetaObjectByOid(ctx, repoID, lfsMetaObject.Oid); errRemove != nil { + return nil, fmt.Errorf("unable to remove failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, errRemove, err) + } + return nil, err } + } + return &lfsMetaObject.Pointer, nil +} + +func checkIsLfsFileInGitAttributes(ctx context.Context, t *TemporaryUploadRepository, paths []string) (ret []bool, err error) { + attributesMap, err := attribute.CheckAttributes(ctx, t.gitRepo, "" /* use temp repo's working dir */, attribute.CheckAttributeOpts{ + Attributes: []string{attribute.Filter}, + Filenames: paths, + }) + if err != nil { + return nil, err + } + for _, p := range paths { + isLFSFile := attributesMap[p] != nil && attributesMap[p].Get(attribute.Filter).ToString().Value() == "lfs" + ret = append(ret, isLFSFile) + } + return ret, nil +} - if filename2attribute2info[file.Options.treePath] != nil && filename2attribute2info[file.Options.treePath]["filter"] == "lfs" { - // OK so we are supposed to LFS this data! - pointer, err := lfs.GeneratePointer(treeObjectContentReader) +type writeRepoObjectRet struct { + ObjectHash string + LfsContent io.ReadCloser // if not nil, then the caller should store its content in LfsPointer, then close it + LfsPointer lfs.Pointer +} + +// writeRepoObjectForModify hashes the git object for create or update operations +func writeRepoObjectForModify(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile) (ret *writeRepoObjectRet, err error) { + ret = &writeRepoObjectRet{} + treeObjectContentReader := file.ContentReader + if setting.LFS.StartServer { + checkIsLfsFiles, err := checkIsLfsFileInGitAttributes(ctx, t, []string{file.Options.treePath}) + if err != nil { + return nil, err + } + if checkIsLfsFiles[0] { + // OK, so we are supposed to LFS this data! + ret.LfsPointer, err = lfs.GeneratePointer(file.ContentReader) if err != nil { - return err + return nil, err } - lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repoID} - treeObjectContentReader = strings.NewReader(pointer.StringContent()) + if _, err = file.ContentReader.Seek(0, io.SeekStart); err != nil { + return nil, err + } + ret.LfsContent = io.NopCloser(file.ContentReader) + treeObjectContentReader = strings.NewReader(ret.LfsPointer.StringContent()) } } - // Add the object to the database - objectHash, err := t.HashObject(ctx, treeObjectContentReader) + ret.ObjectHash, err = t.HashObjectAndWrite(ctx, treeObjectContentReader) if err != nil { - return err + return nil, err } + return ret, nil +} - // Add the object to the index - if file.Options.executable { - if err := t.AddObjectToIndex(ctx, "100755", objectHash, file.Options.treePath); err != nil { - return err - } - } else { - if err := t.AddObjectToIndex(ctx, "100644", objectHash, file.Options.treePath); err != nil { - return err +// writeRepoObjectForRename the same as writeRepoObjectForModify buf for "rename" +func writeRepoObjectForRename(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile) (ret *writeRepoObjectRet, err error) { + lastCommitID, err := t.GetLastCommit(ctx) + if err != nil { + return nil, err + } + commit, err := t.GetCommit(lastCommitID) + if err != nil { + return nil, err + } + oldEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) + if err != nil { + return nil, err + } + + ret = &writeRepoObjectRet{ObjectHash: oldEntry.ID.String()} + if !setting.LFS.StartServer { + return ret, nil + } + + checkIsLfsFiles, err := checkIsLfsFileInGitAttributes(ctx, t, []string{file.Options.fromTreePath, file.Options.treePath}) + if err != nil { + return nil, err + } + oldIsLfs, newIsLfs := checkIsLfsFiles[0], checkIsLfsFiles[1] + + // If the old and new paths are both in lfs or both not in lfs, the object hash of the old file can be used directly + // as the object doesn't change + if oldIsLfs == newIsLfs { + return ret, nil + } + + oldEntryBlobPointerBy := func(f func(r io.Reader) (lfs.Pointer, error)) (lfsPointer lfs.Pointer, err error) { + r, err := oldEntry.Blob().DataAsync() + if err != nil { + return lfsPointer, err } + defer r.Close() + return f(r) } - if lfsMetaObject != nil { - // We have an LFS object - create it - lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, lfsMetaObject.RepositoryID, lfsMetaObject.Pointer) + var treeObjectContentReader io.ReadCloser + if oldIsLfs { + // If the old is in lfs but the new isn't, read the content from lfs and add it as a normal git object + pointer, err := oldEntryBlobPointerBy(lfs.ReadPointer) if err != nil { - return err + return nil, err } - exist, err := contentStore.Exists(lfsMetaObject.Pointer) + treeObjectContentReader, err = lfs.ReadMetaObject(pointer) if err != nil { - return err + return nil, err } - if !exist { - _, err := file.ContentReader.Seek(0, io.SeekStart) - if err != nil { - return err - } - if err := contentStore.Put(lfsMetaObject.Pointer, file.ContentReader); err != nil { - if _, err2 := git_model.RemoveLFSMetaObjectByOid(ctx, repoID, lfsMetaObject.Oid); err2 != nil { - return fmt.Errorf("unable to remove failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, err2, err) - } - return err - } + defer treeObjectContentReader.Close() + } else { + // If the new is in lfs but the old isn't, read the content from the git object and generate a lfs pointer of it + ret.LfsPointer, err = oldEntryBlobPointerBy(lfs.GeneratePointer) + if err != nil { + return nil, err + } + ret.LfsContent, err = oldEntry.Blob().DataAsync() + if err != nil { + return nil, err } + treeObjectContentReader = io.NopCloser(strings.NewReader(ret.LfsPointer.StringContent())) } - - return nil + ret.ObjectHash, err = t.HashObjectAndWrite(ctx, treeObjectContentReader) + if err != nil { + return nil, err + } + return ret, nil } // VerifyBranchProtection verify the branch protection for modifying the given treePath on the given branch diff --git a/services/repository/files/upload.go b/services/repository/files/upload.go index 2e4ed1744e..b783cbd01d 100644 --- a/services/repository/files/upload.go +++ b/services/repository/files/upload.go @@ -8,14 +8,11 @@ import ( "fmt" "os" "path" - "strings" + "sync" - git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/lfs" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/log" ) // UploadRepoFileOptions contains the uploaded repository file options @@ -31,202 +28,84 @@ type UploadRepoFileOptions struct { Committer *IdentityOptions } -type uploadInfo struct { - upload *repo_model.Upload - lfsMetaObject *git_model.LFSMetaObject +type lazyLocalFileReader struct { + *os.File + localFilename string + counter int + mu sync.Mutex } -func cleanUpAfterFailure(ctx context.Context, infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error { - for _, info := range *infos { - if info.lfsMetaObject == nil { - continue - } - if !info.lfsMetaObject.Existing { - if _, err := git_model.RemoveLFSMetaObjectByOid(ctx, t.repo.ID, info.lfsMetaObject.Oid); err != nil { - original = fmt.Errorf("%w, %v", original, err) // We wrap the original error - as this is the underlying error that required the fallback - } - } - } - return original -} - -// UploadRepoFiles uploads files to the given repository -func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *UploadRepoFileOptions) error { - if len(opts.Files) == 0 { - return nil - } +var _ LazyReadSeeker = (*lazyLocalFileReader)(nil) - uploads, err := repo_model.GetUploadsByUUIDs(ctx, opts.Files) - if err != nil { - return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %w", opts.Files, err) - } +func (l *lazyLocalFileReader) Close() error { + l.mu.Lock() + defer l.mu.Unlock() - names := make([]string, len(uploads)) - infos := make([]uploadInfo, len(uploads)) - for i, upload := range uploads { - // Check file is not lfs locked, will return nil if lock setting not enabled - filepath := path.Join(opts.TreePath, upload.Name) - lfsLock, err := git_model.GetTreePathLock(ctx, repo.ID, filepath) - if err != nil { - return err - } - if lfsLock != nil && lfsLock.OwnerID != doer.ID { - u, err := user_model.GetUserByID(ctx, lfsLock.OwnerID) - if err != nil { - return err + if l.counter > 0 { + l.counter-- + if l.counter == 0 { + if err := l.File.Close(); err != nil { + return fmt.Errorf("close file %s: %w", l.localFilename, err) } - return git_model.ErrLFSFileLocked{RepoID: repo.ID, Path: filepath, UserName: u.Name} - } - - names[i] = upload.Name - infos[i] = uploadInfo{upload: upload} - } - - t, err := NewTemporaryUploadRepository(repo) - if err != nil { - return err - } - defer t.Close() - - hasOldBranch := true - if err = t.Clone(ctx, opts.OldBranch, true); err != nil { - if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { - return err - } - if err = t.Init(ctx, repo.ObjectFormatName); err != nil { - return err - } - hasOldBranch = false - opts.LastCommitID = "" - } - if hasOldBranch { - if err = t.SetDefaultIndex(ctx); err != nil { - return err - } - } - - var filename2attribute2info map[string]map[string]string - if setting.LFS.StartServer { - filename2attribute2info, err = t.gitRepo.CheckAttribute(git.CheckAttributeOpts{ - Attributes: []string{"filter"}, - Filenames: names, - CachedOnly: true, - }) - if err != nil { - return err + l.File = nil } + return nil } + return fmt.Errorf("file %s already closed", l.localFilename) +} - // Copy uploaded files into repository. - for i := range infos { - if err := copyUploadedLFSFileIntoRepository(ctx, &infos[i], filename2attribute2info, t, opts.TreePath); err != nil { - return err - } - } +func (l *lazyLocalFileReader) OpenLazyReader() error { + l.mu.Lock() + defer l.mu.Unlock() - // Now write the tree - treeHash, err := t.WriteTree(ctx) - if err != nil { - return err + if l.File != nil { + l.counter++ + return nil } - // Now commit the tree - commitOpts := &CommitTreeUserOptions{ - ParentCommitID: opts.LastCommitID, - TreeHash: treeHash, - CommitMessage: opts.Message, - SignOff: opts.Signoff, - DoerUser: doer, - AuthorIdentity: opts.Author, - CommitterIdentity: opts.Committer, - } - commitHash, err := t.CommitTree(ctx, commitOpts) + file, err := os.Open(l.localFilename) if err != nil { return err } + l.File = file + l.counter = 1 + return nil +} - // Now deal with LFS objects - for i := range infos { - if infos[i].lfsMetaObject == nil { - continue - } - infos[i].lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, infos[i].lfsMetaObject.RepositoryID, infos[i].lfsMetaObject.Pointer) - if err != nil { - // OK Now we need to cleanup - return cleanUpAfterFailure(ctx, &infos, t, err) - } - // Don't move the files yet - we need to ensure that - // everything can be inserted first - } - - // OK now we can insert the data into the store - there's no way to clean up the store - // once it's in there, it's in there. - contentStore := lfs.NewContentStore() - for _, info := range infos { - if err := uploadToLFSContentStore(info, contentStore); err != nil { - return cleanUpAfterFailure(ctx, &infos, t, err) - } - } - - // Then push this tree to NewBranch - if err := t.Push(ctx, doer, commitHash, opts.NewBranch); err != nil { - return err +// UploadRepoFiles uploads files to the given repository +func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *UploadRepoFileOptions) error { + if len(opts.Files) == 0 { + return nil } - return repo_model.DeleteUploads(ctx, uploads...) -} - -func copyUploadedLFSFileIntoRepository(ctx context.Context, info *uploadInfo, filename2attribute2info map[string]map[string]string, t *TemporaryUploadRepository, treePath string) error { - file, err := os.Open(info.upload.LocalPath()) + uploads, err := repo_model.GetUploadsByUUIDs(ctx, opts.Files) if err != nil { - return err + return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %w", opts.Files, err) } - defer file.Close() - var objectHash string - if setting.LFS.StartServer && filename2attribute2info[info.upload.Name] != nil && filename2attribute2info[info.upload.Name]["filter"] == "lfs" { - // Handle LFS - // FIXME: Inefficient! this should probably happen in models.Upload - pointer, err := lfs.GeneratePointer(file) - if err != nil { - return err - } - - info.lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: t.repo.ID} - - if objectHash, err = t.HashObject(ctx, strings.NewReader(pointer.StringContent())); err != nil { - return err - } - } else if objectHash, err = t.HashObject(ctx, file); err != nil { - return err + changeOpts := &ChangeRepoFilesOptions{ + LastCommitID: opts.LastCommitID, + OldBranch: opts.OldBranch, + NewBranch: opts.NewBranch, + Message: opts.Message, + Signoff: opts.Signoff, + Author: opts.Author, + Committer: opts.Committer, + } + for _, upload := range uploads { + changeOpts.Files = append(changeOpts.Files, &ChangeRepoFile{ + Operation: "upload", + TreePath: path.Join(opts.TreePath, upload.Name), + ContentReader: &lazyLocalFileReader{localFilename: upload.LocalPath()}, + }) } - // Add the object to the index - return t.AddObjectToIndex(ctx, "100644", objectHash, path.Join(treePath, info.upload.Name)) -} - -func uploadToLFSContentStore(info uploadInfo, contentStore *lfs.ContentStore) error { - if info.lfsMetaObject == nil { - return nil - } - exist, err := contentStore.Exists(info.lfsMetaObject.Pointer) + _, err = ChangeRepoFiles(ctx, repo, doer, changeOpts) if err != nil { return err } - if !exist { - file, err := os.Open(info.upload.LocalPath()) - if err != nil { - return err - } - - defer file.Close() - // FIXME: Put regenerates the hash and copies the file over. - // I guess this strictly ensures the soundness of the store but this is inefficient. - if err := contentStore.Put(info.lfsMetaObject.Pointer, file); err != nil { - // OK Now we need to cleanup - // Can't clean up the store, once uploaded there they're there. - return err - } + if err := repo_model.DeleteUploads(ctx, uploads...); err != nil { + log.Error("DeleteUploads: %v", err) } return nil } diff --git a/services/repository/fork.go b/services/repository/fork.go index 5b1ba7a418..8bd3498b17 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -65,7 +65,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork } // Fork is prohibited, if user has reached maximum limit of repositories - if !owner.CanForkRepo() { + if !doer.CanForkRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ Limit: owner.MaxRepoCreation, } @@ -100,114 +100,106 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork IsFork: true, ForkID: opts.BaseRepo.ID, ObjectFormatName: opts.BaseRepo.ObjectFormatName, + Status: repo_model.RepositoryBeingMigrated, } - oldRepoPath := opts.BaseRepo.RepoPath() - - needsRollback := false - rollbackFn := func() { - if !needsRollback { - return - } - - if exists, _ := gitrepo.IsRepositoryExist(ctx, repo); !exists { - return - } - - // As the transaction will be failed and hence database changes will be destroyed we only need - // to delete the related repository on the filesystem - if errDelete := gitrepo.DeleteRepository(ctx, repo); errDelete != nil { - log.Error("Failed to remove fork repo") - } - } - - needsRollbackInPanic := true - defer func() { - panicErr := recover() - if panicErr == nil { - return - } - - if needsRollbackInPanic { - rollbackFn() - } - panic(panicErr) - }() - - err = db.WithTx(ctx, func(txCtx context.Context) error { - if err = CreateRepositoryByExample(txCtx, doer, owner, repo, false, true); err != nil { + // 1 - Create the repository in the database + err = db.WithTx(ctx, func(ctx context.Context) error { + if err = createRepositoryInDB(ctx, doer, owner, repo, true); err != nil { return err } - - if err = repo_model.IncrementRepoForkNum(txCtx, opts.BaseRepo.ID); err != nil { + if err = repo_model.IncrementRepoForkNum(ctx, opts.BaseRepo.ID); err != nil { return err } // copy lfs files failure should not be ignored - if err = git_model.CopyLFS(txCtx, repo, opts.BaseRepo); err != nil { - return err - } - - needsRollback = true + return git_model.CopyLFS(ctx, repo, opts.BaseRepo) + }) + if err != nil { + return nil, err + } - cloneCmd := git.NewCommand("clone", "--bare") - if opts.SingleBranch != "" { - cloneCmd.AddArguments("--single-branch", "--branch").AddDynamicArguments(opts.SingleBranch) - } - if stdout, _, err := cloneCmd.AddDynamicArguments(oldRepoPath, repo.RepoPath()). - RunStdBytes(txCtx, &git.RunOpts{Timeout: 10 * time.Minute}); err != nil { - log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, opts.BaseRepo, stdout, err) - return fmt.Errorf("git clone: %w", err) + // last - clean up if something goes wrong + // WARNING: Don't override all later err with local variables + defer func() { + if err != nil { + // we can not use the ctx because it maybe canceled or timeout + cleanupRepository(repo.ID) } + }() - if err := repo_module.CheckDaemonExportOK(txCtx, repo); err != nil { - return fmt.Errorf("checkDaemonExportOK: %w", err) + // 2 - check whether the repository with the same storage exists + var isExist bool + isExist, err = gitrepo.IsRepositoryExist(ctx, repo) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", repo.FullName(), err) + return nil, err + } + if isExist { + log.Error("Files already exist in %s and we are not going to adopt or delete.", repo.FullName()) + // Don't return directly, we need err in defer to cleanupRepository + err = repo_model.ErrRepoFilesAlreadyExist{ + Uname: repo.OwnerName, + Name: repo.Name, } + return nil, err + } - if stdout, _, err := git.NewCommand("update-server-info"). - RunStdString(txCtx, &git.RunOpts{Dir: repo.RepoPath()}); err != nil { - log.Error("Fork Repository (git update-server-info) failed for %v:\nStdout: %s\nError: %v", repo, stdout, err) - return fmt.Errorf("git update-server-info: %w", err) - } + // 3 - Clone the repository + cloneCmd := git.NewCommand("clone", "--bare") + if opts.SingleBranch != "" { + cloneCmd.AddArguments("--single-branch", "--branch").AddDynamicArguments(opts.SingleBranch) + } + var stdout []byte + if stdout, _, err = cloneCmd.AddDynamicArguments(opts.BaseRepo.RepoPath(), repo.RepoPath()). + RunStdBytes(ctx, &git.RunOpts{Timeout: 10 * time.Minute}); err != nil { + log.Error("Fork Repository (git clone) Failed for %v (from %v):\nStdout: %s\nError: %v", repo, opts.BaseRepo, stdout, err) + return nil, fmt.Errorf("git clone: %w", err) + } - if err = gitrepo.CreateDelegateHooks(ctx, repo); err != nil { - return fmt.Errorf("createDelegateHooks: %w", err) - } + // 4 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } - gitRepo, err := gitrepo.OpenRepository(txCtx, repo) - if err != nil { - return fmt.Errorf("OpenRepository: %w", err) - } - defer gitRepo.Close() + // 5 - Create hooks + if err = gitrepo.CreateDelegateHooks(ctx, repo); err != nil { + return nil, fmt.Errorf("createDelegateHooks: %w", err) + } - _, err = repo_module.SyncRepoBranchesWithRepo(txCtx, repo, gitRepo, doer.ID) - return err - }) - needsRollbackInPanic = false + // 6 - Sync the repository branches and tags + var gitRepo *git.Repository + gitRepo, err = gitrepo.OpenRepository(ctx, repo) if err != nil { - rollbackFn() - return nil, err + return nil, fmt.Errorf("OpenRepository: %w", err) } + defer gitRepo.Close() + if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, repo, gitRepo, doer.ID); err != nil { + return nil, fmt.Errorf("SyncRepoBranchesWithRepo: %w", err) + } + if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { + return nil, fmt.Errorf("Sync releases from git tags failed: %v", err) + } + + // 7 - Update the repository // even if below operations failed, it could be ignored. And they will be retried - if err := repo_module.UpdateRepoSize(ctx, repo); err != nil { + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { log.Error("Failed to update size for repository: %v", err) + err = nil } - if err := repo_model.CopyLanguageStat(ctx, opts.BaseRepo, repo); err != nil { + if err = repo_model.CopyLanguageStat(ctx, opts.BaseRepo, repo); err != nil { log.Error("Copy language stat from oldRepo failed: %v", err) + err = nil } - if err := repo_model.CopyLicense(ctx, opts.BaseRepo, repo); err != nil { + if err = repo_model.CopyLicense(ctx, opts.BaseRepo, repo); err != nil { return nil, err } - gitRepo, err := gitrepo.OpenRepository(ctx, repo) - if err != nil { - log.Error("Open created git repository failed: %v", err) - } else { - defer gitRepo.Close() - if err := repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { - log.Error("Sync releases from git tags failed: %v", err) - } + // 8 - update repository status to be ready + repo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } notify_service.ForkRepository(ctx, doer, opts.BaseRepo, repo) @@ -217,7 +209,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork // ConvertForkToNormalRepository convert the provided repo from a forked repo to normal repo func ConvertForkToNormalRepository(ctx context.Context, repo *repo_model.Repository) error { - err := db.WithTx(ctx, func(ctx context.Context) error { + return db.WithTx(ctx, func(ctx context.Context) error { repo, err := repo_model.GetRepositoryByID(ctx, repo.ID) if err != nil { return err @@ -234,16 +226,8 @@ func ConvertForkToNormalRepository(ctx context.Context, repo *repo_model.Reposit repo.IsFork = false repo.ForkID = 0 - - if err := repo_module.UpdateRepository(ctx, repo, false); err != nil { - log.Error("Unable to update repository %-v whilst converting from fork. Error: %v", repo, err) - return err - } - - return nil + return repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_fork", "fork_id") }) - - return err } type findForksOptions struct { diff --git a/services/repository/fork_test.go b/services/repository/fork_test.go index 452798b25b..5375f79028 100644 --- a/services/repository/fork_test.go +++ b/services/repository/fork_test.go @@ -4,13 +4,17 @@ package repository import ( + "os" "testing" + "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" ) @@ -35,7 +39,7 @@ func TestForkRepository(t *testing.T) { assert.False(t, repo_model.IsErrReachLimitOfRepo(err)) // change AllowForkWithoutMaximumLimit to false for the test - setting.Repository.AllowForkWithoutMaximumLimit = false + defer test.MockVariableValue(&setting.Repository.AllowForkWithoutMaximumLimit, false)() // user has reached maximum limit of repositories user.MaxRepoCreation = 0 fork2, err := ForkRepository(git.DefaultContext, user, user, ForkRepoOptions{ @@ -46,3 +50,43 @@ func TestForkRepository(t *testing.T) { assert.Nil(t, fork2) assert.True(t, repo_model.IsErrReachLimitOfRepo(err)) } + +func TestForkRepositoryCleanup(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + // a successful fork + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo10 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}) + + fork, err := ForkRepository(git.DefaultContext, user2, user2, ForkRepoOptions{ + BaseRepo: repo10, + Name: "test", + }) + assert.NoError(t, err) + assert.NotNil(t, fork) + + exist, err := util.IsExist(repo_model.RepoPath(user2.Name, "test")) + assert.NoError(t, err) + assert.True(t, exist) + + err = DeleteRepositoryDirectly(db.DefaultContext, fork.ID) + assert.NoError(t, err) + + // a failed creating because some mock data + // create the repository directory so that the creation will fail after database record created. + assert.NoError(t, os.MkdirAll(repo_model.RepoPath(user2.Name, "test"), os.ModePerm)) + + fork2, err := ForkRepository(db.DefaultContext, user2, user2, ForkRepoOptions{ + BaseRepo: repo10, + Name: "test", + }) + assert.Nil(t, fork2) + assert.Error(t, err) + + // assert the cleanup is successful + unittest.AssertNotExistsBean(t, &repo_model.Repository{OwnerName: user2.Name, Name: "test"}) + + exist, err = util.IsExist(repo_model.RepoPath(user2.Name, "test")) + assert.NoError(t, err) + assert.False(t, exist) +} diff --git a/services/repository/generate.go b/services/repository/generate.go index 9d2bbb1f7f..867b5d7855 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -17,11 +17,11 @@ import ( git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" @@ -42,10 +42,8 @@ type expansion struct { var defaultTransformers = []transformer{ {Name: "SNAKE", Transform: xstrings.ToSnakeCase}, {Name: "KEBAB", Transform: xstrings.ToKebabCase}, - {Name: "CAMEL", Transform: func(str string) string { - return xstrings.FirstRuneToLower(xstrings.ToCamelCase(str)) - }}, - {Name: "PASCAL", Transform: xstrings.ToCamelCase}, + {Name: "CAMEL", Transform: xstrings.ToCamelCase}, + {Name: "PASCAL", Transform: xstrings.ToPascalCase}, {Name: "LOWER", Transform: strings.ToLower}, {Name: "UPPER", Transform: strings.ToUpper}, {Name: "TITLE", Transform: util.ToTitleCase}, @@ -255,48 +253,35 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r return initRepoCommit(ctx, tmpDir, repo, repo.Owner, defaultBranch) } -func generateGitContent(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository) (err error) { - tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name) +// GenerateGitContent generates git content from a template repository +func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) (err error) { + tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-" + generateRepo.Name) if err != nil { - return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.FullName(), err) + return fmt.Errorf("failed to create temp dir for repository %s: %w", generateRepo.FullName(), err) } + defer cleanup() - defer func() { - if err := util.RemoveAll(tmpDir); err != nil { - log.Error("RemoveAll: %v", err) - } - }() - - if err = generateRepoCommit(ctx, repo, templateRepo, generateRepo, tmpDir); err != nil { + if err = generateRepoCommit(ctx, generateRepo, templateRepo, generateRepo, tmpDir); err != nil { return fmt.Errorf("generateRepoCommit: %w", err) } // re-fetch repo - if repo, err = repo_model.GetRepositoryByID(ctx, repo.ID); err != nil { + if generateRepo, err = repo_model.GetRepositoryByID(ctx, generateRepo.ID); err != nil { return fmt.Errorf("getRepositoryByID: %w", err) } // if there was no default branch supplied when generating the repo, use the default one from the template - if strings.TrimSpace(repo.DefaultBranch) == "" { - repo.DefaultBranch = templateRepo.DefaultBranch + if strings.TrimSpace(generateRepo.DefaultBranch) == "" { + generateRepo.DefaultBranch = templateRepo.DefaultBranch } - if err = gitrepo.SetDefaultBranch(ctx, repo, repo.DefaultBranch); err != nil { + if err = gitrepo.SetDefaultBranch(ctx, generateRepo, generateRepo.DefaultBranch); err != nil { return fmt.Errorf("setDefaultBranch: %w", err) } - if err = UpdateRepository(ctx, repo, false); err != nil { + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, generateRepo, "default_branch"); err != nil { return fmt.Errorf("updateRepository: %w", err) } - return nil -} - -// GenerateGitContent generates git content from a template repository -func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error { - if err := generateGitContent(ctx, generateRepo, templateRepo, generateRepo); err != nil { - return err - } - if err := repo_module.UpdateRepoSize(ctx, generateRepo); err != nil { return fmt.Errorf("failed to update size for repository: %w", err) } @@ -328,57 +313,6 @@ func (gro GenerateRepoOptions) IsValid() bool { gro.IssueLabels || gro.ProtectedBranch // or other items as they are added } -// generateRepository generates a repository from a template -func generateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) { - generateRepo := &repo_model.Repository{ - OwnerID: owner.ID, - Owner: owner, - OwnerName: owner.Name, - Name: opts.Name, - LowerName: strings.ToLower(opts.Name), - Description: opts.Description, - DefaultBranch: opts.DefaultBranch, - IsPrivate: opts.Private, - IsEmpty: !opts.GitContent || templateRepo.IsEmpty, - IsFsckEnabled: templateRepo.IsFsckEnabled, - TemplateID: templateRepo.ID, - TrustModel: templateRepo.TrustModel, - ObjectFormatName: templateRepo.ObjectFormatName, - } - - if err = CreateRepositoryByExample(ctx, doer, owner, generateRepo, false, false); err != nil { - return nil, err - } - - isExist, err := gitrepo.IsRepositoryExist(ctx, generateRepo) - if err != nil { - log.Error("Unable to check if %s exists. Error: %v", generateRepo.FullName(), err) - return nil, err - } - if isExist { - return nil, repo_model.ErrRepoFilesAlreadyExist{ - Uname: generateRepo.OwnerName, - Name: generateRepo.Name, - } - } - - if err = repo_module.CheckInitRepository(ctx, generateRepo); err != nil { - return generateRepo, err - } - - if err = repo_module.CheckDaemonExportOK(ctx, generateRepo); err != nil { - return generateRepo, fmt.Errorf("checkDaemonExportOK: %w", err) - } - - if stdout, _, err := git.NewCommand("update-server-info"). - RunStdString(ctx, &git.RunOpts{Dir: generateRepo.RepoPath()}); err != nil { - log.Error("GenerateRepository(git update-server-info) in %v: Stdout: %s\nError: %v", generateRepo, stdout, err) - return generateRepo, fmt.Errorf("error in GenerateRepository(git update-server-info): %w", err) - } - - return generateRepo, nil -} - var fileNameSanitizeRegexp = regexp.MustCompile(`(?i)\.\.|[<>:\"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`) // Sanitize user input to valid OS filenames diff --git a/services/repository/generate_test.go b/services/repository/generate_test.go index b0f97d0ffb..1163c392c9 100644 --- a/services/repository/generate_test.go +++ b/services/repository/generate_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var giteaTemplate = []byte(` @@ -65,3 +66,26 @@ func TestFileNameSanitize(t *testing.T) { assert.Equal(t, "_", fileNameSanitize("\u0000")) assert.Equal(t, "ç›®æ ‡", fileNameSanitize("ç›®æ ‡")) } + +func TestTransformers(t *testing.T) { + cases := []struct { + name string + expected string + }{ + {"SNAKE", "abc_def_xyz"}, + {"KEBAB", "abc-def-xyz"}, + {"CAMEL", "abcDefXyz"}, + {"PASCAL", "AbcDefXyz"}, + {"LOWER", "abc_def-xyz"}, + {"UPPER", "ABC_DEF-XYZ"}, + {"TITLE", "Abc_def-Xyz"}, + } + + input := "Abc_Def-XYZ" + assert.Len(t, defaultTransformers, len(cases)) + for i, c := range cases { + tf := defaultTransformers[i] + require.Equal(t, c.name, tf.Name) + assert.Equal(t, c.expected, tf.Transform(input), "case %s", c.name) + } +} diff --git a/services/repository/gitgraph/graph_models.go b/services/repository/gitgraph/graph_models.go index c45662836b..02b0268cd9 100644 --- a/services/repository/gitgraph/graph_models.go +++ b/services/repository/gitgraph/graph_models.go @@ -121,7 +121,7 @@ func (graph *Graph) LoadAndProcessCommits(ctx context.Context, repository *repo_ return repo_model.IsOwnerMemberCollaborator(ctx, repository, user.ID) }, &keyMap) - statuses, _, err := git_model.GetLatestCommitStatus(ctx, repository.ID, c.Commit.ID.String(), db.ListOptions{}) + statuses, err := git_model.GetLatestCommitStatus(ctx, repository.ID, c.Commit.ID.String(), db.ListOptionsAll) if err != nil { log.Error("GetLatestCommitStatus: %v", err) } else { @@ -232,8 +232,8 @@ func newRefsFromRefNames(refNames []byte) []git.Reference { continue } refName := string(refNameBytes) - if strings.HasPrefix(refName, "tag: ") { - refName = strings.TrimPrefix(refName, "tag: ") + if after, ok := strings.CutPrefix(refName, "tag: "); ok { + refName = after } else { refName = strings.TrimPrefix(refName, "HEAD -> ") } diff --git a/services/repository/gitgraph/graph_test.go b/services/repository/gitgraph/graph_test.go index 4c48b94aa2..93fa1aec6a 100644 --- a/services/repository/gitgraph/graph_test.go +++ b/services/repository/gitgraph/graph_test.go @@ -6,6 +6,7 @@ package gitgraph import ( "bytes" "fmt" + "slices" "strings" "testing" @@ -117,13 +118,7 @@ func TestReleaseUnusedColors(t *testing.T) { if parser.firstAvailable == -1 { // All in use for _, color := range parser.availableColors { - found := false - for _, oldColor := range parser.oldColors { - if oldColor == color { - found = true - break - } - } + found := slices.Contains(parser.oldColors, color) if !found { t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should be available but is not", testcase.availableColors, @@ -141,13 +136,7 @@ func TestReleaseUnusedColors(t *testing.T) { // Some in use for i := parser.firstInUse; i != parser.firstAvailable; i = (i + 1) % len(parser.availableColors) { color := parser.availableColors[i] - found := false - for _, oldColor := range parser.oldColors { - if oldColor == color { - found = true - break - } - } + found := slices.Contains(parser.oldColors, color) if !found { t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should be available but is not", testcase.availableColors, @@ -163,13 +152,7 @@ func TestReleaseUnusedColors(t *testing.T) { } for i := parser.firstAvailable; i != parser.firstInUse; i = (i + 1) % len(parser.availableColors) { color := parser.availableColors[i] - found := false - for _, oldColor := range parser.oldColors { - if oldColor == color { - found = true - break - } - } + found := slices.Contains(parser.oldColors, color) if found { t.Errorf("In testcase:\n%d\t%d\t%d %d =>\n%d\t%d\t%d %d: %d should not be available but is", testcase.availableColors, diff --git a/services/repository/init.go b/services/repository/init.go index bd777b8a2f..1eeeb4aa4f 100644 --- a/services/repository/init.go +++ b/services/repository/init.go @@ -42,9 +42,12 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi cmd := git.NewCommand("commit", "--message=Initial commit"). AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email) - sign, keyID, signer, _ := asymkey_service.SignInitialCommit(ctx, tmpPath, u) + sign, key, signer, _ := asymkey_service.SignInitialCommit(ctx, tmpPath, u) if sign { - cmd.AddOptionFormat("-S%s", keyID) + if key.Format != "" { + cmd.AddConfig("gpg.format", key.Format) + } + cmd.AddOptionFormat("-S%s", key.KeyID) if repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { // need to set the committer to the KeyID owner diff --git a/services/repository/license_test.go b/services/repository/license_test.go index 9e74a268f5..eb897f3c03 100644 --- a/services/repository/license_test.go +++ b/services/repository/license_test.go @@ -4,7 +4,6 @@ package repository import ( - "fmt" "strings" "testing" @@ -45,7 +44,7 @@ func Test_detectLicense(t *testing.T) { assert.NoError(t, err) tests = append(tests, DetectLicenseTest{ - name: fmt.Sprintf("single license test: %s", licenseName), + name: "single license test: " + licenseName, arg: string(license), want: []string{licenseName}, }) diff --git a/services/repository/merge_upstream.go b/services/repository/merge_upstream.go index 34e01df723..8d6f11372c 100644 --- a/services/repository/merge_upstream.go +++ b/services/repository/merge_upstream.go @@ -18,7 +18,7 @@ import ( ) // MergeUpstream merges the base repository's default branch into the fork repository's current branch. -func MergeUpstream(ctx reqctx.RequestContext, doer *user_model.User, repo *repo_model.Repository, branch string) (mergeStyle string, err error) { +func MergeUpstream(ctx reqctx.RequestContext, doer *user_model.User, repo *repo_model.Repository, branch string, ffOnly bool) (mergeStyle string, err error) { if err = repo.MustNotBeArchived(); err != nil { return "", err } @@ -45,6 +45,11 @@ func MergeUpstream(ctx reqctx.RequestContext, doer *user_model.User, repo *repo_ return "", err } + // If ff_only is requested and fast-forward failed, return error + if ffOnly { + return "", util.NewInvalidArgumentErrorf("fast-forward merge not possible: branch has diverged") + } + // TODO: FakePR: it is somewhat hacky, but it is the only way to "merge" at the moment // ideally in the future the "merge" functions should be refactored to decouple from the PullRequest fakeIssue := &issue_model.Issue{ diff --git a/services/repository/migrate.go b/services/repository/migrate.go index 5b6feccb8d..0a3dc45339 100644 --- a/services/repository/migrate.go +++ b/services/repository/migrate.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/organization" repo_model "code.gitea.io/gitea/models/repo" + unit_model "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" @@ -117,14 +118,8 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, repo.Owner = u } - if err := repo_module.CheckDaemonExportOK(ctx, repo); err != nil { - return repo, fmt.Errorf("checkDaemonExportOK: %w", err) - } - - if stdout, _, err := git.NewCommand("update-server-info"). - RunStdString(ctx, &git.RunOpts{Dir: repoPath}); err != nil { - log.Error("MigrateRepositoryGitData(git update-server-info) in %v: Stdout: %s\nError: %v", repo, stdout, err) - return repo, fmt.Errorf("error in MigrateRepositoryGitData(git update-server-info): %w", err) + if err := updateGitRepoAfterCreate(ctx, repo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) } gitRepo, err := git.OpenRepository(ctx, repoPath) @@ -141,12 +136,12 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, if !repo.IsEmpty { if len(repo.DefaultBranch) == 0 { // Try to get HEAD branch and set it as default branch. - headBranch, err := gitRepo.GetHEADBranch() + headBranchName, err := git.GetDefaultBranch(ctx, repoPath) if err != nil { return repo, fmt.Errorf("GetHEADBranch: %w", err) } - if headBranch != nil { - repo.DefaultBranch = headBranch.Name + if headBranchName != "" { + repo.DefaultBranch = headBranchName } } @@ -154,9 +149,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err) } + // if releases migration are not requested, we will sync all tags here + // otherwise, the releases sync will be done out of this function if !opts.Releases { - // note: this will greatly improve release (tag) sync - // for pull-mirrors with many tags repo.IsMirror = opts.Mirror if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { log.Error("Failed to synchronize tags to releases for repository: %v", err) @@ -225,10 +220,14 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } repo.IsMirror = true - if err = UpdateRepository(ctx, repo, false); err != nil { + if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "num_watches", "is_empty", "default_branch", "default_wiki_branch", "is_mirror"); err != nil { return nil, err } + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + // this is necessary for sync local tags from remote configName := fmt.Sprintf("remote.%s.fetch", mirrorModel.GetRemoteName()) if stdout, _, err := git.NewCommand("config"). @@ -246,6 +245,19 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } } + var enableRepoUnits []repo_model.RepoUnit + if opts.Releases && !unit_model.TypeReleases.UnitGlobalDisabled() { + enableRepoUnits = append(enableRepoUnits, repo_model.RepoUnit{RepoID: repo.ID, Type: unit_model.TypeReleases}) + } + if opts.Wiki && !unit_model.TypeWiki.UnitGlobalDisabled() { + enableRepoUnits = append(enableRepoUnits, repo_model.RepoUnit{RepoID: repo.ID, Type: unit_model.TypeWiki}) + } + if len(enableRepoUnits) > 0 { + err = UpdateRepositoryUnits(ctx, repo, enableRepoUnits, nil) + if err != nil { + return nil, err + } + } return repo, committer.Commit() } diff --git a/services/repository/push.go b/services/repository/push.go index 6d3b9dd252..7c68a7f176 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -66,7 +66,7 @@ func PushUpdates(opts []*repo_module.PushUpdateOptions) error { for _, opt := range opts { if opt.IsNewRef() && opt.IsDelRef() { - return fmt.Errorf("Old and new revisions are both NULL") + return errors.New("Old and new revisions are both NULL") } } @@ -232,7 +232,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { } if len(addTags)+len(delTags) > 0 { - if err := PushUpdateAddDeleteTags(ctx, repo, gitRepo, addTags, delTags); err != nil { + if err := PushUpdateAddDeleteTags(ctx, repo, gitRepo, pusher, addTags, delTags); err != nil { return fmt.Errorf("PushUpdateAddDeleteTags: %w", err) } } @@ -283,7 +283,7 @@ func pushNewBranch(ctx context.Context, repo *repo_model.Repository, pusher *use } } // Update the is empty and default_branch columns - if err := repo_model.UpdateRepositoryCols(ctx, repo, "default_branch", "is_empty"); err != nil { + if err := repo_model.UpdateRepositoryColsWithAutoTime(ctx, repo, "default_branch", "is_empty"); err != nil { return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } } @@ -342,17 +342,17 @@ func pushDeleteBranch(ctx context.Context, repo *repo_model.Repository, pusher * } // PushUpdateAddDeleteTags updates a number of added and delete tags -func PushUpdateAddDeleteTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, addTags, delTags []string) error { +func PushUpdateAddDeleteTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, pusher *user_model.User, addTags, delTags []string) error { return db.WithTx(ctx, func(ctx context.Context) error { - if err := repo_model.PushUpdateDeleteTagsContext(ctx, repo, delTags); err != nil { + if err := repo_model.PushUpdateDeleteTags(ctx, repo, delTags); err != nil { return err } - return pushUpdateAddTags(ctx, repo, gitRepo, addTags) + return pushUpdateAddTags(ctx, repo, gitRepo, pusher, addTags) }) } // pushUpdateAddTags updates a number of add tags -func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tags []string) error { +func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, pusher *user_model.User, tags []string) error { if len(tags) == 0 { return nil } @@ -378,14 +378,12 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo newReleases := make([]*repo_model.Release, 0, len(lowerTags)-len(relMap)) - emailToUser := make(map[string]*user_model.User) - for i, lowerTag := range lowerTags { tag, err := gitRepo.GetTag(tags[i]) if err != nil { return fmt.Errorf("GetTag: %w", err) } - commit, err := tag.Commit(gitRepo) + commit, err := gitRepo.GetTagCommit(tag.Name) if err != nil { return fmt.Errorf("Commit: %w", err) } @@ -397,69 +395,42 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo if sig == nil { sig = commit.Committer } - var author *user_model.User - createdAt := time.Unix(1, 0) + createdAt := time.Unix(1, 0) if sig != nil { - var ok bool - author, ok = emailToUser[sig.Email] - if !ok { - author, err = user_model.GetUserByEmail(ctx, sig.Email) - if err != nil && !user_model.IsErrUserNotExist(err) { - return fmt.Errorf("GetUserByEmail: %w", err) - } - if author != nil { - emailToUser[sig.Email] = author - } - } createdAt = sig.When } - commitsCount, err := commit.CommitsCount() - if err != nil { - return fmt.Errorf("CommitsCount: %w", err) - } - rel, has := relMap[lowerTag] - - parts := strings.SplitN(tag.Message, "\n", 2) - note := "" - if len(parts) > 1 { - note = parts[1] - } + title, note := git.SplitCommitTitleBody(tag.Message, 255) if !has { rel = &repo_model.Release{ RepoID: repo.ID, - Title: parts[0], + Title: title, TagName: tags[i], LowerTagName: lowerTag, Target: "", Sha1: commit.ID.String(), - NumCommits: commitsCount, + NumCommits: -1, // the commits count will be updated when the UI needs it Note: note, IsDraft: false, IsPrerelease: false, IsTag: true, + PublisherID: pusher.ID, CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), } - if author != nil { - rel.PublisherID = author.ID - } newReleases = append(newReleases, rel) } else { rel.Sha1 = commit.ID.String() rel.CreatedUnix = timeutil.TimeStamp(createdAt.Unix()) - rel.NumCommits = commitsCount if rel.IsTag { - rel.Title = parts[0] + rel.Title = title rel.Note = note - if author != nil { - rel.PublisherID = author.ID - } } else { rel.IsDraft = false } + rel.PublisherID = pusher.ID if err = repo_model.UpdateRelease(ctx, rel); err != nil { return fmt.Errorf("Update: %w", err) } diff --git a/services/repository/repo_team.go b/services/repository/repo_team.go index 672ee49fea..8ea186f8cc 100644 --- a/services/repository/repo_team.go +++ b/services/repository/repo_team.go @@ -86,17 +86,9 @@ func RemoveAllRepositoriesFromTeam(ctx context.Context, t *organization.Team) (e return nil } - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - if err = removeAllRepositoriesFromTeam(ctx, t); err != nil { - return err - } - - return committer.Commit() + return db.WithTx(ctx, func(ctx context.Context) error { + return removeAllRepositoriesFromTeam(ctx, t) + }) } // removeAllRepositoriesFromTeam removes all repositories from team and recalculates access @@ -167,17 +159,9 @@ func RemoveRepositoryFromTeam(ctx context.Context, t *organization.Team, repoID return err } - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - if err = removeRepositoryFromTeam(ctx, t, repo, true); err != nil { - return err - } - - return committer.Commit() + return db.WithTx(ctx, func(ctx context.Context) error { + return removeRepositoryFromTeam(ctx, t, repo, true) + }) } // removeRepositoryFromTeam removes a repository from a team and recalculates access diff --git a/services/repository/repository.go b/services/repository/repository.go index fcc617979e..e574dc6c01 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -5,22 +5,29 @@ package repository import ( "context" + "errors" "fmt" + "os" + "path/filepath" + "strings" + activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/organization" + access_model "code.gitea.io/gitea/models/perm/access" repo_model "code.gitea.io/gitea/models/repo" - system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/graceful" + issue_indexer "code.gitea.io/gitea/modules/indexer/issues" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/queue" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" pull_service "code.gitea.io/gitea/services/pull" ) @@ -40,7 +47,7 @@ type WebSearchResults struct { // CreateRepository creates a repository for the user/organization. func CreateRepository(ctx context.Context, doer, owner *user_model.User, opts CreateRepoOptions) (*repo_model.Repository, error) { - repo, err := CreateRepositoryDirectly(ctx, doer, owner, opts) + repo, err := CreateRepositoryDirectly(ctx, doer, owner, opts, true) if err != nil { // No need to rollback here we should do this in CreateRepository... return nil, err @@ -62,7 +69,7 @@ func DeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_mod notify_service.DeleteRepository(ctx, doer, repo) } - return DeleteRepositoryDirectly(ctx, doer, repo.ID) + return DeleteRepositoryDirectly(ctx, repo.ID) } // PushCreateRepo creates a repository when a new repository is pushed to an appropriate namespace @@ -72,10 +79,10 @@ func PushCreateRepo(ctx context.Context, authUser, owner *user_model.User, repoN if ok, err := organization.CanCreateOrgRepo(ctx, owner.ID, authUser.ID); err != nil { return nil, err } else if !ok { - return nil, fmt.Errorf("cannot push-create repository for org") + return nil, errors.New("cannot push-create repository for org") } } else if authUser.ID != owner.ID { - return nil, fmt.Errorf("cannot push-create repository for another user") + return nil, errors.New("cannot push-create repository for another user") } } @@ -94,15 +101,13 @@ func PushCreateRepo(ctx context.Context, authUser, owner *user_model.User, repoN func Init(ctx context.Context) error { licenseUpdaterQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_license_updater", repoLicenseUpdater) if licenseUpdaterQueue == nil { - return fmt.Errorf("unable to create repo_license_updater queue") + return errors.New("unable to create repo_license_updater queue") } go graceful.GetManager().RunWithCancel(licenseUpdaterQueue) if err := repo_module.LoadRepoConfig(); err != nil { return err } - system_model.RemoveAllWithNotice(ctx, "Clean up temporary repository uploads", setting.Repository.Upload.TempPath) - system_model.RemoveAllWithNotice(ctx, "Clean up temporary repositories", repo_module.LocalCopyPath()) if err := initPushQueue(); err != nil { return err } @@ -111,42 +116,107 @@ func Init(ctx context.Context) error { // UpdateRepository updates a repository func UpdateRepository(ctx context.Context, repo *repo_model.Repository, visibilityChanged bool) (err error) { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - if err = repo_module.UpdateRepository(ctx, repo, visibilityChanged); err != nil { - return fmt.Errorf("updateRepository: %w", err) - } - - return committer.Commit() + return db.WithTx(ctx, func(ctx context.Context) error { + if err = updateRepository(ctx, repo, visibilityChanged); err != nil { + return fmt.Errorf("updateRepository: %w", err) + } + return nil + }) } -func UpdateRepositoryVisibility(ctx context.Context, repo *repo_model.Repository, isPrivate bool) (err error) { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } +func MakeRepoPublic(ctx context.Context, repo *repo_model.Repository) (err error) { + return db.WithTx(ctx, func(ctx context.Context) error { + repo.IsPrivate = false + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_private"); err != nil { + return err + } - defer committer.Close() + if err = repo.LoadOwner(ctx); err != nil { + return fmt.Errorf("LoadOwner: %w", err) + } + if repo.Owner.IsOrganization() { + // Organization repository need to recalculate access table when visibility is changed. + if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { + return fmt.Errorf("recalculateTeamAccesses: %w", err) + } + } - repo.IsPrivate = isPrivate + // Create/Remove git-daemon-export-ok for git-daemon... + if err := checkDaemonExportOK(ctx, repo); err != nil { + return err + } - if err = repo_module.UpdateRepository(ctx, repo, true); err != nil { - return fmt.Errorf("UpdateRepositoryVisibility: %w", err) - } + forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) + if err != nil { + return fmt.Errorf("getRepositoriesByForkID: %w", err) + } - return committer.Commit() -} + if repo.Owner.Visibility != structs.VisibleTypePrivate { + for i := range forkRepos { + if err = MakeRepoPublic(ctx, forkRepos[i]); err != nil { + return fmt.Errorf("MakeRepoPublic[%d]: %w", forkRepos[i].ID, err) + } + } + } -func MakeRepoPublic(ctx context.Context, repo *repo_model.Repository) (err error) { - return UpdateRepositoryVisibility(ctx, repo, false) + // If visibility is changed, we need to update the issue indexer. + // Since the data in the issue indexer have field to indicate if the repo is public or not. + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + + return nil + }) } func MakeRepoPrivate(ctx context.Context, repo *repo_model.Repository) (err error) { - return UpdateRepositoryVisibility(ctx, repo, true) + return db.WithTx(ctx, func(ctx context.Context) error { + repo.IsPrivate = true + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "is_private"); err != nil { + return err + } + + if err = repo.LoadOwner(ctx); err != nil { + return fmt.Errorf("LoadOwner: %w", err) + } + if repo.Owner.IsOrganization() { + // Organization repository need to recalculate access table when visibility is changed. + if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { + return fmt.Errorf("recalculateTeamAccesses: %w", err) + } + } + + // If repo has become private, we need to set its actions to private. + _, err = db.GetEngine(ctx).Where("repo_id = ?", repo.ID).Cols("is_private").Update(&activities_model.Action{ + IsPrivate: true, + }) + if err != nil { + return err + } + + if err = repo_model.ClearRepoStars(ctx, repo.ID); err != nil { + return err + } + + // Create/Remove git-daemon-export-ok for git-daemon... + if err := checkDaemonExportOK(ctx, repo); err != nil { + return err + } + + forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) + if err != nil { + return fmt.Errorf("getRepositoriesByForkID: %w", err) + } + for i := range forkRepos { + if err = MakeRepoPrivate(ctx, forkRepos[i]); err != nil { + return fmt.Errorf("MakeRepoPrivate[%d]: %w", forkRepos[i].ID, err) + } + } + + // If visibility is changed, we need to update the issue indexer. + // Since the data in the issue indexer have field to indicate if the repo is public or not. + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + + return nil + }) } // LinkedRepository returns the linked repo if any @@ -172,3 +242,97 @@ func LinkedRepository(ctx context.Context, a *repo_model.Attachment) (*repo_mode } return nil, -1, nil } + +// checkDaemonExportOK creates/removes git-daemon-export-ok for git-daemon... +func checkDaemonExportOK(ctx context.Context, repo *repo_model.Repository) error { + if err := repo.LoadOwner(ctx); err != nil { + return err + } + + // Create/Remove git-daemon-export-ok for git-daemon... + daemonExportFile := filepath.Join(repo.RepoPath(), `git-daemon-export-ok`) + + isExist, err := util.IsExist(daemonExportFile) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err) + return err + } + + isPublic := !repo.IsPrivate && repo.Owner.Visibility == structs.VisibleTypePublic + if !isPublic && isExist { + if err = util.Remove(daemonExportFile); err != nil { + log.Error("Failed to remove %s: %v", daemonExportFile, err) + } + } else if isPublic && !isExist { + if f, err := os.Create(daemonExportFile); err != nil { + log.Error("Failed to create %s: %v", daemonExportFile, err) + } else { + f.Close() + } + } + + return nil +} + +// updateRepository updates a repository with db context +func updateRepository(ctx context.Context, repo *repo_model.Repository, visibilityChanged bool) (err error) { + repo.LowerName = strings.ToLower(repo.Name) + + e := db.GetEngine(ctx) + + if _, err = e.ID(repo.ID).NoAutoTime().AllCols().Update(repo); err != nil { + return fmt.Errorf("update: %w", err) + } + + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + + if visibilityChanged { + if err = repo.LoadOwner(ctx); err != nil { + return fmt.Errorf("LoadOwner: %w", err) + } + if repo.Owner.IsOrganization() { + // Organization repository need to recalculate access table when visibility is changed. + if err = access_model.RecalculateTeamAccesses(ctx, repo, 0); err != nil { + return fmt.Errorf("recalculateTeamAccesses: %w", err) + } + } + + // If repo has become private, we need to set its actions to private. + if repo.IsPrivate { + _, err = e.Where("repo_id = ?", repo.ID).Cols("is_private").Update(&activities_model.Action{ + IsPrivate: true, + }) + if err != nil { + return err + } + + if err = repo_model.ClearRepoStars(ctx, repo.ID); err != nil { + return err + } + } + + // Create/Remove git-daemon-export-ok for git-daemon... + if err := checkDaemonExportOK(ctx, repo); err != nil { + return err + } + + forkRepos, err := repo_model.GetRepositoriesByForkID(ctx, repo.ID) + if err != nil { + return fmt.Errorf("getRepositoriesByForkID: %w", err) + } + for i := range forkRepos { + forkRepos[i].IsPrivate = repo.IsPrivate || repo.Owner.Visibility == structs.VisibleTypePrivate + if err = updateRepository(ctx, forkRepos[i], true); err != nil { + return fmt.Errorf("updateRepository[%d]: %w", forkRepos[i].ID, err) + } + } + + // If visibility is changed, we need to update the issue indexer. + // Since the data in the issue indexer have field to indicate if the repo is public or not. + issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + } + + return nil +} diff --git a/services/repository/repository_test.go b/services/repository/repository_test.go index 892a11a23e..8f9fdf8fa1 100644 --- a/services/repository/repository_test.go +++ b/services/repository/repository_test.go @@ -6,6 +6,7 @@ package repository import ( "testing" + activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" @@ -40,3 +41,23 @@ func TestLinkedRepository(t *testing.T) { }) } } + +func TestUpdateRepositoryVisibilityChanged(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + // Get sample repo and change visibility + repo, err := repo_model.GetRepositoryByID(db.DefaultContext, 9) + assert.NoError(t, err) + repo.IsPrivate = true + + // Update it + err = updateRepository(db.DefaultContext, repo, true) + assert.NoError(t, err) + + // Check visibility of action has become private + act := activities_model.Action{} + _, err = db.GetEngine(db.DefaultContext).ID(3).Get(&act) + + assert.NoError(t, err) + assert.True(t, act.IsPrivate) +} diff --git a/services/repository/setting.go b/services/repository/setting.go index e0c787dd2d..b6873691eb 100644 --- a/services/repository/setting.go +++ b/services/repository/setting.go @@ -16,41 +16,37 @@ import ( // UpdateRepositoryUnits updates a repository's units func UpdateRepositoryUnits(ctx context.Context, repo *repo_model.Repository, units []repo_model.RepoUnit, deleteUnitTypes []unit.Type) (err error) { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - - // Delete existing settings of units before adding again - for _, u := range units { - deleteUnitTypes = append(deleteUnitTypes, u.Type) - } - - if slices.Contains(deleteUnitTypes, unit.TypeActions) { - if err := actions_service.CleanRepoScheduleTasks(ctx, repo); err != nil { - log.Error("CleanRepoScheduleTasks: %v", err) + return db.WithTx(ctx, func(ctx context.Context) error { + // Delete existing settings of units before adding again + for _, u := range units { + deleteUnitTypes = append(deleteUnitTypes, u.Type) } - } - for _, u := range units { - if u.Type == unit.TypeActions { - if err := actions_service.DetectAndHandleSchedules(ctx, repo); err != nil { - log.Error("DetectAndHandleSchedules: %v", err) + if slices.Contains(deleteUnitTypes, unit.TypeActions) { + if err := actions_service.CleanRepoScheduleTasks(ctx, repo); err != nil { + log.Error("CleanRepoScheduleTasks: %v", err) } - break } - } - if _, err = db.GetEngine(ctx).Where("repo_id = ?", repo.ID).In("type", deleteUnitTypes).Delete(new(repo_model.RepoUnit)); err != nil { - return err - } + for _, u := range units { + if u.Type == unit.TypeActions { + if err := actions_service.DetectAndHandleSchedules(ctx, repo); err != nil { + log.Error("DetectAndHandleSchedules: %v", err) + } + break + } + } - if len(units) > 0 { - if err = db.Insert(ctx, units); err != nil { + if _, err = db.GetEngine(ctx).Where("repo_id = ?", repo.ID).In("type", deleteUnitTypes).Delete(new(repo_model.RepoUnit)); err != nil { return err } - } - return committer.Commit() + if len(units) > 0 { + if err = db.Insert(ctx, units); err != nil { + return err + } + } + + return nil + }) } diff --git a/services/repository/template.go b/services/repository/template.go index 36a680c8e2..6906a60083 100644 --- a/services/repository/template.go +++ b/services/repository/template.go @@ -5,12 +5,17 @@ package repository import ( "context" + "fmt" + "strings" "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" issues_model "code.gitea.io/gitea/models/issues" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" notify_service "code.gitea.io/gitea/services/notify" ) @@ -63,70 +68,124 @@ func GenerateProtectedBranch(ctx context.Context, templateRepo, generateRepo *re // GenerateRepository generates a repository from a template func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templateRepo *repo_model.Repository, opts GenerateRepoOptions) (_ *repo_model.Repository, err error) { - if !doer.IsAdmin && !owner.CanCreateRepo() { + if !doer.CanCreateRepoIn(owner) { return nil, repo_model.ErrReachLimitOfRepo{ Limit: owner.MaxRepoCreation, } } - var generateRepo *repo_model.Repository - if err = db.WithTx(ctx, func(ctx context.Context) error { - generateRepo, err = generateRepository(ctx, doer, owner, templateRepo, opts) + generateRepo := &repo_model.Repository{ + OwnerID: owner.ID, + Owner: owner, + OwnerName: owner.Name, + Name: opts.Name, + LowerName: strings.ToLower(opts.Name), + Description: opts.Description, + DefaultBranch: opts.DefaultBranch, + IsPrivate: opts.Private, + IsEmpty: !opts.GitContent || templateRepo.IsEmpty, + IsFsckEnabled: templateRepo.IsFsckEnabled, + TemplateID: templateRepo.ID, + TrustModel: templateRepo.TrustModel, + ObjectFormatName: templateRepo.ObjectFormatName, + Status: repo_model.RepositoryBeingMigrated, + } + + // 1 - Create the repository in the database + if err := db.WithTx(ctx, func(ctx context.Context) error { + return createRepositoryInDB(ctx, doer, owner, generateRepo, false) + }); err != nil { + return nil, err + } + + // last - clean up the repository if something goes wrong + defer func() { if err != nil { - return err + // we can not use the ctx because it maybe canceled or timeout + cleanupRepository(generateRepo.ID) } + }() - // Git Content - if opts.GitContent && !templateRepo.IsEmpty { - if err = GenerateGitContent(ctx, templateRepo, generateRepo); err != nil { - return err - } + // 2 - check whether the repository with the same storage exists + isExist, err := gitrepo.IsRepositoryExist(ctx, generateRepo) + if err != nil { + log.Error("Unable to check if %s exists. Error: %v", generateRepo.FullName(), err) + return nil, err + } + if isExist { + // Don't return directly, we need err in defer to cleanupRepository + err = repo_model.ErrRepoFilesAlreadyExist{ + Uname: generateRepo.OwnerName, + Name: generateRepo.Name, } + return nil, err + } - // Topics - if opts.Topics { - if err = repo_model.GenerateTopics(ctx, templateRepo, generateRepo); err != nil { - return err - } + // 3 -Init git bare new repository. + if err = git.InitRepository(ctx, generateRepo.RepoPath(), true, generateRepo.ObjectFormatName); err != nil { + return nil, fmt.Errorf("git.InitRepository: %w", err) + } else if err = gitrepo.CreateDelegateHooks(ctx, generateRepo); err != nil { + return nil, fmt.Errorf("createDelegateHooks: %w", err) + } + + // 4 - Update the git repository + if err = updateGitRepoAfterCreate(ctx, generateRepo); err != nil { + return nil, fmt.Errorf("updateGitRepoAfterCreate: %w", err) + } + + // 5 - generate the repository contents according to the template + // Git Content + if opts.GitContent && !templateRepo.IsEmpty { + if err = GenerateGitContent(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Git Hooks - if opts.GitHooks { - if err = GenerateGitHooks(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Topics + if opts.Topics { + if err = repo_model.GenerateTopics(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Webhooks - if opts.Webhooks { - if err = GenerateWebhooks(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Git Hooks + if opts.GitHooks { + if err = GenerateGitHooks(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Avatar - if opts.Avatar && len(templateRepo.Avatar) > 0 { - if err = generateAvatar(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Webhooks + if opts.Webhooks { + if err = GenerateWebhooks(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - // Issue Labels - if opts.IssueLabels { - if err = GenerateIssueLabels(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Avatar + if opts.Avatar && len(templateRepo.Avatar) > 0 { + if err = generateAvatar(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - if opts.ProtectedBranch { - if err = GenerateProtectedBranch(ctx, templateRepo, generateRepo); err != nil { - return err - } + // Issue Labels + if opts.IssueLabels { + if err = GenerateIssueLabels(ctx, templateRepo, generateRepo); err != nil { + return nil, err } + } - return nil - }); err != nil { - return nil, err + if opts.ProtectedBranch { + if err = GenerateProtectedBranch(ctx, templateRepo, generateRepo); err != nil { + return nil, err + } + } + + // 6 - update repository status to be ready + generateRepo.Status = repo_model.RepositoryReady + if err = repo_model.UpdateRepositoryColsWithAutoTime(ctx, generateRepo, "status"); err != nil { + return nil, fmt.Errorf("UpdateRepositoryCols: %w", err) } notify_service.CreateRepository(ctx, doer, owner, generateRepo) diff --git a/services/repository/transfer.go b/services/repository/transfer.go index a589bc469d..5ad63cca67 100644 --- a/services/repository/transfer.go +++ b/services/repository/transfer.go @@ -20,10 +20,22 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/globallock" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" ) +type LimitReachedError struct{ Limit int } + +func (LimitReachedError) Error() string { + return "Repository limit has been reached" +} + +func IsRepositoryLimitReached(err error) bool { + _, ok := err.(LimitReachedError) + return ok +} + func getRepoWorkingLockKey(repoID int64) string { return fmt.Sprintf("repo_working_%d", repoID) } @@ -49,6 +61,11 @@ func AcceptTransferOwnership(ctx context.Context, repo *repo_model.Repository, d return err } + if !doer.CanCreateRepoIn(repoTransfer.Recipient) { + limit := util.Iif(repoTransfer.Recipient.MaxRepoCreation >= 0, repoTransfer.Recipient.MaxRepoCreation, setting.Repository.MaxCreationLimit) + return LimitReachedError{Limit: limit} + } + if !repoTransfer.CanUserAcceptOrRejectTransfer(ctx, doer) { return util.ErrPermissionDenied } @@ -143,7 +160,7 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName repo.OwnerName = newOwner.Name // Update repository. - if err := repo_model.UpdateRepositoryCols(ctx, repo, "owner_id", "owner_name"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "owner_id", "owner_name"); err != nil { return fmt.Errorf("update owner: %w", err) } @@ -287,7 +304,7 @@ func transferOwnership(ctx context.Context, doer *user_model.User, newOwnerName return fmt.Errorf("deleteRepositoryTransfer: %w", err) } repo.Status = repo_model.RepositoryReady - if err := repo_model.UpdateRepositoryCols(ctx, repo, "status"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "status"); err != nil { return err } @@ -399,6 +416,11 @@ func StartRepositoryTransfer(ctx context.Context, doer, newOwner *user_model.Use return err } + if !doer.CanForkRepoIn(newOwner) { + limit := util.Iif(newOwner.MaxRepoCreation >= 0, newOwner.MaxRepoCreation, setting.Repository.MaxCreationLimit) + return LimitReachedError{Limit: limit} + } + var isDirectTransfer bool oldOwnerName := repo.OwnerName @@ -473,7 +495,7 @@ func RejectRepositoryTransfer(ctx context.Context, repo *repo_model.Repository, } repo.Status = repo_model.RepositoryReady - if err := repo_model.UpdateRepositoryCols(ctx, repo, "status"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repo, "status"); err != nil { return err } @@ -521,7 +543,7 @@ func CancelRepositoryTransfer(ctx context.Context, repoTransfer *repo_model.Repo } repoTransfer.Repo.Status = repo_model.RepositoryReady - if err := repo_model.UpdateRepositoryCols(ctx, repoTransfer.Repo, "status"); err != nil { + if err := repo_model.UpdateRepositoryColsNoAutoTime(ctx, repoTransfer.Repo, "status"); err != nil { return err } diff --git a/services/repository/transfer_test.go b/services/repository/transfer_test.go index 16a8fb6e1e..80a073e9f9 100644 --- a/services/repository/transfer_test.go +++ b/services/repository/transfer_test.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package repository @@ -14,11 +14,14 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/feed" notify_service "code.gitea.io/gitea/services/notify" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var notifySync sync.Once @@ -125,3 +128,40 @@ func TestRepositoryTransfer(t *testing.T) { err = RejectRepositoryTransfer(db.DefaultContext, repo2, doer) assert.True(t, repo_model.IsErrNoPendingTransfer(err)) } + +// Test transfer rejections +func TestRepositoryTransferRejection(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + // Set limit to 0 repositories so no repositories can be transferred + defer test.MockVariableValue(&setting.Repository.MaxCreationLimit, 0)() + + // Admin case + doerAdmin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 5}) + + transfer, err := repo_model.GetPendingRepositoryTransfer(db.DefaultContext, repo) + require.NoError(t, err) + require.NotNil(t, transfer) + require.NoError(t, transfer.LoadRecipient(db.DefaultContext)) + + require.True(t, doerAdmin.CanCreateRepoIn(transfer.Recipient)) // admin is not subject to limits + + // Administrator should not be affected by the limits so transfer should be successful + assert.NoError(t, AcceptTransferOwnership(db.DefaultContext, repo, doerAdmin)) + + // Non admin user case + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 10}) + repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 21}) + + transfer, err = repo_model.GetPendingRepositoryTransfer(db.DefaultContext, repo) + require.NoError(t, err) + require.NotNil(t, transfer) + require.NoError(t, transfer.LoadRecipient(db.DefaultContext)) + + require.False(t, doer.CanCreateRepoIn(transfer.Recipient)) // regular user is subject to limits + + // Cannot accept because of the limit + err = AcceptTransferOwnership(db.DefaultContext, repo, doer) + assert.Error(t, err) + assert.True(t, IsRepositoryLimitReached(err)) +} |