summaryrefslogtreecommitdiffstats
path: root/modules/indexer
diff options
context:
space:
mode:
authorzeripath <art27@cantab.net>2022-01-19 23:26:57 +0000
committerGitHub <noreply@github.com>2022-01-19 23:26:57 +0000
commit5cb0c9aa0d7eed087055b1efca79628957207d36 (patch)
treed117a514e1f17e5f6bfcda1be273f6a971112663 /modules/indexer
parent4563148a61ba892e8f2bb66342f00a950bcd5315 (diff)
downloadgitea-5cb0c9aa0d7eed087055b1efca79628957207d36.tar.gz
gitea-5cb0c9aa0d7eed087055b1efca79628957207d36.zip
Propagate context and ensure git commands run in request context (#17868)
This PR continues the work in #17125 by progressively ensuring that git commands run within the request context. This now means that the if there is a git repo already open in the context it will be used instead of reopening it. Signed-off-by: Andrew Thornton <art27@cantab.net>
Diffstat (limited to 'modules/indexer')
-rw-r--r--modules/indexer/code/bleve.go13
-rw-r--r--modules/indexer/code/elastic_search.go10
-rw-r--r--modules/indexer/code/git.go23
-rw-r--r--modules/indexer/code/indexer.go12
-rw-r--r--modules/indexer/code/indexer_test.go3
-rw-r--r--modules/indexer/code/wrapped.go5
6 files changed, 35 insertions, 31 deletions
diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go
index 3a44a8dc05..64e974dc18 100644
--- a/modules/indexer/code/bleve.go
+++ b/modules/indexer/code/bleve.go
@@ -6,6 +6,7 @@ package code
import (
"bufio"
+ "context"
"fmt"
"io"
"os"
@@ -180,7 +181,7 @@ func NewBleveIndexer(indexDir string) (*BleveIndexer, bool, error) {
return indexer, created, err
}
-func (b *BleveIndexer) addUpdate(batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string,
+func (b *BleveIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, commitSha string,
update fileUpdate, repo *repo_model.Repository, batch *gitea_bleve.FlushingBatch) error {
// Ignore vendored files in code search
if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
@@ -190,7 +191,7 @@ func (b *BleveIndexer) addUpdate(batchWriter git.WriteCloserError, batchReader *
size := update.Size
if !update.Sized {
- stdout, err := git.NewCommand("cat-file", "-s", update.BlobSha).
+ stdout, err := git.NewCommandContext(ctx, "cat-file", "-s", update.BlobSha).
RunInDir(repo.RepoPath())
if err != nil {
return err
@@ -271,21 +272,21 @@ func (b *BleveIndexer) Close() {
}
// Index indexes the data
-func (b *BleveIndexer) Index(repo *repo_model.Repository, sha string, changes *repoChanges) error {
+func (b *BleveIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
if len(changes.Updates) > 0 {
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
- if err := git.EnsureValidGitRepository(git.DefaultContext, repo.RepoPath()); err != nil {
+ if err := git.EnsureValidGitRepository(ctx, repo.RepoPath()); err != nil {
log.Error("Unable to open git repo: %s for %-v: %v", repo.RepoPath(), repo, err)
return err
}
- batchWriter, batchReader, cancel := git.CatFileBatch(git.DefaultContext, repo.RepoPath())
+ batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repo.RepoPath())
defer cancel()
for _, update := range changes.Updates {
- if err := b.addUpdate(batchWriter, batchReader, sha, update, repo, batch); err != nil {
+ if err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo, batch); err != nil {
return err
}
}
diff --git a/modules/indexer/code/elastic_search.go b/modules/indexer/code/elastic_search.go
index 169dffd78b..eb57d4d68a 100644
--- a/modules/indexer/code/elastic_search.go
+++ b/modules/indexer/code/elastic_search.go
@@ -177,7 +177,7 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
return exists, nil
}
-func (b *ElasticSearchIndexer) addUpdate(batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update fileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) {
+func (b *ElasticSearchIndexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserError, batchReader *bufio.Reader, sha string, update fileUpdate, repo *repo_model.Repository) ([]elastic.BulkableRequest, error) {
// Ignore vendored files in code search
if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
return nil, nil
@@ -186,7 +186,7 @@ func (b *ElasticSearchIndexer) addUpdate(batchWriter git.WriteCloserError, batch
size := update.Size
if !update.Sized {
- stdout, err := git.NewCommand("cat-file", "-s", update.BlobSha).
+ stdout, err := git.NewCommandContext(ctx, "cat-file", "-s", update.BlobSha).
RunInDir(repo.RepoPath())
if err != nil {
return nil, err
@@ -244,7 +244,7 @@ func (b *ElasticSearchIndexer) addDelete(filename string, repo *repo_model.Repos
}
// Index will save the index data
-func (b *ElasticSearchIndexer) Index(repo *repo_model.Repository, sha string, changes *repoChanges) error {
+func (b *ElasticSearchIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
reqs := make([]elastic.BulkableRequest, 0)
if len(changes.Updates) > 0 {
// Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first!
@@ -253,11 +253,11 @@ func (b *ElasticSearchIndexer) Index(repo *repo_model.Repository, sha string, ch
return err
}
- batchWriter, batchReader, cancel := git.CatFileBatch(git.DefaultContext, repo.RepoPath())
+ batchWriter, batchReader, cancel := git.CatFileBatch(ctx, repo.RepoPath())
defer cancel()
for _, update := range changes.Updates {
- updateReqs, err := b.addUpdate(batchWriter, batchReader, sha, update, repo)
+ updateReqs, err := b.addUpdate(ctx, batchWriter, batchReader, sha, update, repo)
if err != nil {
return err
}
diff --git a/modules/indexer/code/git.go b/modules/indexer/code/git.go
index b76f6d6778..247354054c 100644
--- a/modules/indexer/code/git.go
+++ b/modules/indexer/code/git.go
@@ -5,6 +5,7 @@
package code
import (
+ "context"
"strconv"
"strings"
@@ -27,8 +28,8 @@ type repoChanges struct {
RemovedFilenames []string
}
-func getDefaultBranchSha(repo *repo_model.Repository) (string, error) {
- stdout, err := git.NewCommand("show-ref", "-s", git.BranchPrefix+repo.DefaultBranch).RunInDir(repo.RepoPath())
+func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (string, error) {
+ stdout, err := git.NewCommandContext(ctx, "show-ref", "-s", git.BranchPrefix+repo.DefaultBranch).RunInDir(repo.RepoPath())
if err != nil {
return "", err
}
@@ -36,16 +37,16 @@ func getDefaultBranchSha(repo *repo_model.Repository) (string, error) {
}
// getRepoChanges returns changes to repo since last indexer update
-func getRepoChanges(repo *repo_model.Repository, revision string) (*repoChanges, error) {
+func getRepoChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) {
status, err := repo_model.GetIndexerStatus(repo, repo_model.RepoIndexerTypeCode)
if err != nil {
return nil, err
}
if len(status.CommitSha) == 0 {
- return genesisChanges(repo, revision)
+ return genesisChanges(ctx, repo, revision)
}
- return nonGenesisChanges(repo, revision)
+ return nonGenesisChanges(ctx, repo, revision)
}
func isIndexable(entry *git.TreeEntry) bool {
@@ -89,9 +90,9 @@ func parseGitLsTreeOutput(stdout []byte) ([]fileUpdate, error) {
}
// genesisChanges get changes to add repo to the indexer for the first time
-func genesisChanges(repo *repo_model.Repository, revision string) (*repoChanges, error) {
+func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) {
var changes repoChanges
- stdout, err := git.NewCommand("ls-tree", "--full-tree", "-l", "-r", revision).
+ stdout, err := git.NewCommandContext(ctx, "ls-tree", "--full-tree", "-l", "-r", revision).
RunInDirBytes(repo.RepoPath())
if err != nil {
return nil, err
@@ -101,8 +102,8 @@ func genesisChanges(repo *repo_model.Repository, revision string) (*repoChanges,
}
// nonGenesisChanges get changes since the previous indexer update
-func nonGenesisChanges(repo *repo_model.Repository, revision string) (*repoChanges, error) {
- diffCmd := git.NewCommand("diff", "--name-status",
+func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revision string) (*repoChanges, error) {
+ diffCmd := git.NewCommandContext(ctx, "diff", "--name-status",
repo.CodeIndexerStatus.CommitSha, revision)
stdout, err := diffCmd.RunInDir(repo.RepoPath())
if err != nil {
@@ -112,7 +113,7 @@ func nonGenesisChanges(repo *repo_model.Repository, revision string) (*repoChang
if err = indexer.Delete(repo.ID); err != nil {
return nil, err
}
- return genesisChanges(repo, revision)
+ return genesisChanges(ctx, repo, revision)
}
var changes repoChanges
updatedFilenames := make([]string, 0, 10)
@@ -166,7 +167,7 @@ func nonGenesisChanges(repo *repo_model.Repository, revision string) (*repoChang
}
}
- cmd := git.NewCommand("ls-tree", "--full-tree", "-l", revision, "--")
+ cmd := git.NewCommandContext(ctx, "ls-tree", "--full-tree", "-l", revision, "--")
cmd.AddArguments(updatedFilenames...)
lsTreeStdout, err := cmd.RunInDirBytes(repo.RepoPath())
if err != nil {
diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go
index a616d0e662..a17484cae5 100644
--- a/modules/indexer/code/indexer.go
+++ b/modules/indexer/code/indexer.go
@@ -42,7 +42,7 @@ type SearchResultLanguages struct {
// Indexer defines an interface to index and search code contents
type Indexer interface {
- Index(repo *repo_model.Repository, sha string, changes *repoChanges) error
+ Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error
Delete(repoID int64) error
Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
Close()
@@ -82,7 +82,7 @@ var (
indexerQueue queue.UniqueQueue
)
-func index(indexer Indexer, repoID int64) error {
+func index(ctx context.Context, indexer Indexer, repoID int64) error {
repo, err := repo_model.GetRepositoryByID(repoID)
if repo_model.IsErrRepoNotExist(err) {
return indexer.Delete(repoID)
@@ -91,18 +91,18 @@ func index(indexer Indexer, repoID int64) error {
return err
}
- sha, err := getDefaultBranchSha(repo)
+ sha, err := getDefaultBranchSha(ctx, repo)
if err != nil {
return err
}
- changes, err := getRepoChanges(repo, sha)
+ changes, err := getRepoChanges(ctx, repo, sha)
if err != nil {
return err
} else if changes == nil {
return nil
}
- if err := indexer.Index(repo, sha, changes); err != nil {
+ if err := indexer.Index(ctx, repo, sha, changes); err != nil {
return err
}
@@ -150,7 +150,7 @@ func Init() {
}
log.Trace("IndexerData Process Repo: %d", indexerData.RepoID)
- if err := index(indexer, indexerData.RepoID); err != nil {
+ if err := index(ctx, indexer, indexerData.RepoID); err != nil {
log.Error("index: %v", err)
continue
}
diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go
index 42f21ed0df..71d58c08c5 100644
--- a/modules/indexer/code/indexer_test.go
+++ b/modules/indexer/code/indexer_test.go
@@ -9,6 +9,7 @@ import (
"testing"
"code.gitea.io/gitea/models/unittest"
+ "code.gitea.io/gitea/modules/git"
_ "code.gitea.io/gitea/models"
@@ -22,7 +23,7 @@ func TestMain(m *testing.M) {
func testIndexer(name string, t *testing.T, indexer Indexer) {
t.Run(name, func(t *testing.T) {
var repoID int64 = 1
- err := index(indexer, repoID)
+ err := index(git.DefaultContext, indexer, repoID)
assert.NoError(t, err)
var (
keywords = []struct {
diff --git a/modules/indexer/code/wrapped.go b/modules/indexer/code/wrapped.go
index 7e97a6a324..e7b5b89eef 100644
--- a/modules/indexer/code/wrapped.go
+++ b/modules/indexer/code/wrapped.go
@@ -5,6 +5,7 @@
package code
import (
+ "context"
"fmt"
"sync"
@@ -57,12 +58,12 @@ func (w *wrappedIndexer) get() (Indexer, error) {
return w.internal, nil
}
-func (w *wrappedIndexer) Index(repo *repo_model.Repository, sha string, changes *repoChanges) error {
+func (w *wrappedIndexer) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error {
indexer, err := w.get()
if err != nil {
return err
}
- return indexer.Index(repo, sha, changes)
+ return indexer.Index(ctx, repo, sha, changes)
}
func (w *wrappedIndexer) Delete(repoID int64) error {