diff options
author | zeripath <art27@cantab.net> | 2022-01-19 23:26:57 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-01-19 23:26:57 +0000 |
commit | 5cb0c9aa0d7eed087055b1efca79628957207d36 (patch) | |
tree | d117a514e1f17e5f6bfcda1be273f6a971112663 /modules/indexer | |
parent | 4563148a61ba892e8f2bb66342f00a950bcd5315 (diff) | |
download | gitea-5cb0c9aa0d7eed087055b1efca79628957207d36.tar.gz gitea-5cb0c9aa0d7eed087055b1efca79628957207d36.zip |
Propagate context and ensure git commands run in request context (#17868)
This PR continues the work in #17125 by progressively ensuring that git
commands run within the request context.
This now means that the if there is a git repo already open in the context it will be used instead of reopening it.
Signed-off-by: Andrew Thornton <art27@cantab.net>
Diffstat (limited to 'modules/indexer')
-rw-r--r-- | modules/indexer/code/bleve.go | 13 | ||||
-rw-r--r-- | modules/indexer/code/elastic_search.go | 10 | ||||
-rw-r--r-- | modules/indexer/code/git.go | 23 | ||||
-rw-r--r-- | modules/indexer/code/indexer.go | 12 | ||||
-rw-r--r-- | modules/indexer/code/indexer_test.go | 3 | ||||
-rw-r--r-- | modules/indexer/code/wrapped.go | 5 |
6 files changed, 35 insertions, 31 deletions
diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go index 3a44a8dc05..64e974dc18 100644 --- a/modules/indexer/code/bleve.go +++ b/modules/indexer/code/bleve.go @@ -6,6 +6,7 @@ package code import ( "bufio" + "context" "fmt" "io" "os" @@ -180,7 +181,7 @@ func NewBleveIndexer(indexDir string) (*BleveIndexer, bool, error) { return indexer, created, err } -func (b *BleveIndexer) addUpdate(batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string, +func (b *BleveIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string, update fileUpdate, repo *repo_model.Repository, batch *gitea_bleve.FlushingBatch) error { // Ignore vendored files in code search if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { @@ -190,7 +191,7 @@ func (b *BleveIndexer) addUpdate(batchWriter git.WriteCloserError, batchReader * size := update.Size if !update.Sized { - stdout, err := git.NewCommand("cat-file", "-s", update.BlobSha). + stdout, err := git.NewCommandContext(ctx, "cat-file", "-s", update.BlobSha). RunInDir(repo.RepoPath()) if err != nil { return err @@ -271,21 +272,21 @@ func (b *BleveIndexer) Close() { } // Index indexes the data -func (b *BleveIndexer) Index(repo *repo_model.Repository, sha string, changes *repoChanges) error { +func (b *BleveIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error { batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize) if len(changes.Updates) > 0 { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := git.EnsureValidGitRepository(git.DefaultContext, repo.RepoPath()); err != nil { + if err := git.EnsureValidGitRepository(ctx, repo.RepoPath()); err != nil { log.Error("Unable to open git repo: %s for %-v: %v", repo.RepoPath(), repo, err) return err } - batchWriter, batchReader, cancel := git.CatFileBatch(git.DefaultContext, repo.RepoPath()) + batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repo.RepoPath()) defer cancel() for _, update := range changes.Updates { - if err := b.addUpdate(batchWriter, batchReader, sha, update, repo, batch); err != nil { + if err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo, batch); err != nil { return err } } diff --git a/modules/indexer/code/elastic_search.go b/modules/indexer/code/elastic_search.go index 169dffd78b..eb57d4d68a 100644 --- a/modules/indexer/code/elastic_search.go +++ b/modules/indexer/code/elastic_search.go @@ -177,7 +177,7 @@ func (b *ElasticSearchIndexer) init() (bool, error) { return exists, nil } -func (b *ElasticSearchIndexer) addUpdate(batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update fileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { +func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update fileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) { // Ignore vendored files in code search if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) { return nil, nil @@ -186,7 +186,7 @@ func (b *ElasticSearchIndexer) addUpdate(batchWriter git.WriteCloserError, batch size := update.Size if !update.Sized { - stdout, err := git.NewCommand("cat-file", "-s", update.BlobSha). + stdout, err := git.NewCommandContext(ctx, "cat-file", "-s", update.BlobSha). RunInDir(repo.RepoPath()) if err != nil { return nil, err @@ -244,7 +244,7 @@ func (b *ElasticSearchIndexer) addDelete(filename string, repo *repo_model.Repos } // Index will save the index data -func (b *ElasticSearchIndexer) Index(repo *repo_model.Repository, sha string, changes *repoChanges) error { +func (b *ElasticSearchIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error { reqs := make([]elastic.BulkableRequest, 0) if len(changes.Updates) > 0 { // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! @@ -253,11 +253,11 @@ func (b *ElasticSearchIndexer) Index(repo *repo_model.Repository, sha string, ch return err } - batchWriter, batchReader, cancel := git.CatFileBatch(git.DefaultContext, repo.RepoPath()) + batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repo.RepoPath()) defer cancel() for _, update := range changes.Updates { - updateReqs, err := b.addUpdate(batchWriter, batchReader, sha, update, repo) + updateReqs, err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo) if err != nil { return err } diff --git a/modules/indexer/code/git.go b/modules/indexer/code/git.go index b76f6d6778..247354054c 100644 --- a/modules/indexer/code/git.go +++ b/modules/indexer/code/git.go @@ -5,6 +5,7 @@ package code import ( + "context" "strconv" "strings" @@ -27,8 +28,8 @@ type repoChanges struct { RemovedFilenames []string } -func getDefaultBranchSha(repo *repo_model.Repository) (string, error) { - stdout, err := git.NewCommand("show-ref", "-s", git.BranchPrefix+repo.DefaultBranch).RunInDir(repo.RepoPath()) +func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (string, error) { + stdout, err := git.NewCommandContext(ctx, "show-ref", "-s", git.BranchPrefix+repo.DefaultBranch).RunInDir(repo.RepoPath()) if err != nil { return "", err } @@ -36,16 +37,16 @@ func getDefaultBranchSha(repo *repo_model.Repository) (string, error) { } // getRepoChanges returns changes to repo since last indexer update -func getRepoChanges(repo *repo_model.Repository, revision string) (*repoChanges, error) { +func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) { status, err := repo_model.GetIndexerStatus(repo, repo_model.RepoIndexerTypeCode) if err != nil { return nil, err } if len(status.CommitSha) == 0 { - return genesisChanges(repo, revision) + return genesisChanges(ctx, repo, revision) } - return nonGenesisChanges(repo, revision) + return nonGenesisChanges(ctx, repo, revision) } func isIndexable(entry *git.TreeEntry) bool { @@ -89,9 +90,9 @@ func parseGitLsTreeOutput(stdout []byte) ([]fileUpdate, error) { } // genesisChanges get changes to add repo to the indexer for the first time -func genesisChanges(repo *repo_model.Repository, revision string) (*repoChanges, error) { +func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) { var changes repoChanges - stdout, err := git.NewCommand("ls-tree", "--full-tree", "-l", "-r", revision). + stdout, err := git.NewCommandContext(ctx, "ls-tree", "--full-tree", "-l", "-r", revision). RunInDirBytes(repo.RepoPath()) if err != nil { return nil, err @@ -101,8 +102,8 @@ func genesisChanges(repo *repo_model.Repository, revision string) (*repoChanges, } // nonGenesisChanges get changes since the previous indexer update -func nonGenesisChanges(repo *repo_model.Repository, revision string) (*repoChanges, error) { - diffCmd := git.NewCommand("diff", "--name-status", +func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) { + diffCmd := git.NewCommandContext(ctx, "diff", "--name-status", repo.CodeIndexerStatus.CommitSha, revision) stdout, err := diffCmd.RunInDir(repo.RepoPath()) if err != nil { @@ -112,7 +113,7 @@ func nonGenesisChanges(repo *repo_model.Repository, revision string) (*repoChang if err = indexer.Delete(repo.ID); err != nil { return nil, err } - return genesisChanges(repo, revision) + return genesisChanges(ctx, repo, revision) } var changes repoChanges updatedFilenames := make([]string, 0, 10) @@ -166,7 +167,7 @@ func nonGenesisChanges(repo *repo_model.Repository, revision string) (*repoChang } } - cmd := git.NewCommand("ls-tree", "--full-tree", "-l", revision, "--") + cmd := git.NewCommandContext(ctx, "ls-tree", "--full-tree", "-l", revision, "--") cmd.AddArguments(updatedFilenames...) lsTreeStdout, err := cmd.RunInDirBytes(repo.RepoPath()) if err != nil { diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index a616d0e662..a17484cae5 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -42,7 +42,7 @@ type SearchResultLanguages struct { // Indexer defines an interface to index and search code contents type Indexer interface { - Index(repo *repo_model.Repository, sha string, changes *repoChanges) error + Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error Delete(repoID int64) error Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) Close() @@ -82,7 +82,7 @@ var ( indexerQueue queue.UniqueQueue ) -func index(indexer Indexer, repoID int64) error { +func index(ctx context.Context, indexer Indexer, repoID int64) error { repo, err := repo_model.GetRepositoryByID(repoID) if repo_model.IsErrRepoNotExist(err) { return indexer.Delete(repoID) @@ -91,18 +91,18 @@ func index(indexer Indexer, repoID int64) error { return err } - sha, err := getDefaultBranchSha(repo) + sha, err := getDefaultBranchSha(ctx, repo) if err != nil { return err } - changes, err := getRepoChanges(repo, sha) + changes, err := getRepoChanges(ctx, repo, sha) if err != nil { return err } else if changes == nil { return nil } - if err := indexer.Index(repo, sha, changes); err != nil { + if err := indexer.Index(ctx, repo, sha, changes); err != nil { return err } @@ -150,7 +150,7 @@ func Init() { } log.Trace("IndexerData Process Repo: %d", indexerData.RepoID) - if err := index(indexer, indexerData.RepoID); err != nil { + if err := index(ctx, indexer, indexerData.RepoID); err != nil { log.Error("index: %v", err) continue } diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index 42f21ed0df..71d58c08c5 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -9,6 +9,7 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/git" _ "code.gitea.io/gitea/models" @@ -22,7 +23,7 @@ func TestMain(m *testing.M) { func testIndexer(name string, t *testing.T, indexer Indexer) { t.Run(name, func(t *testing.T) { var repoID int64 = 1 - err := index(indexer, repoID) + err := index(git.DefaultContext, indexer, repoID) assert.NoError(t, err) var ( keywords = []struct { diff --git a/modules/indexer/code/wrapped.go b/modules/indexer/code/wrapped.go index 7e97a6a324..e7b5b89eef 100644 --- a/modules/indexer/code/wrapped.go +++ b/modules/indexer/code/wrapped.go @@ -5,6 +5,7 @@ package code import ( + "context" "fmt" "sync" @@ -57,12 +58,12 @@ func (w *wrappedIndexer) get() (Indexer, error) { return w.internal, nil } -func (w *wrappedIndexer) Index(repo *repo_model.Repository, sha string, changes *repoChanges) error { +func (w *wrappedIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error { indexer, err := w.get() if err != nil { return err } - return indexer.Index(repo, sha, changes) + return indexer.Index(ctx, repo, sha, changes) } func (w *wrappedIndexer) Delete(repoID int64) error { |