aboutsummaryrefslogtreecommitdiffstats
path: root/modules/indexer
diff options
context:
space:
mode:
Diffstat (limited to 'modules/indexer')
-rw-r--r--modules/indexer/code/bleve/token/path/path.go8
-rw-r--r--modules/indexer/code/git.go4
-rw-r--r--modules/indexer/code/search.go2
-rw-r--r--modules/indexer/issues/db/db.go17
-rw-r--r--modules/indexer/issues/db/options.go8
-rw-r--r--modules/indexer/issues/dboptions.go13
-rw-r--r--modules/indexer/issues/indexer.go13
-rw-r--r--modules/indexer/issues/internal/tests/tests.go31
-rw-r--r--modules/indexer/stats/db.go3
9 files changed, 56 insertions, 43 deletions
diff --git a/modules/indexer/code/bleve/token/path/path.go b/modules/indexer/code/bleve/token/path/path.go
index 19f4b5bb5e..6dfc12f146 100644
--- a/modules/indexer/code/bleve/token/path/path.go
+++ b/modules/indexer/code/bleve/token/path/path.go
@@ -51,7 +51,7 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke
slices.Reverse(input)
}
- for i := 0; i < len(input); i++ {
+ for i := range input {
var sb strings.Builder
sb.Write(input[0].Term)
@@ -97,5 +97,9 @@ func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.Toke
}
func init() {
- registry.RegisterTokenFilter(Name, TokenFilterConstructor)
+ // FIXME: move it to the bleve's init function, but do not call it in global init
+ err := registry.RegisterTokenFilter(Name, TokenFilterConstructor)
+ if err != nil {
+ panic(err)
+ }
}
diff --git a/modules/indexer/code/git.go b/modules/indexer/code/git.go
index 0089dd259f..41bc74e6ec 100644
--- a/modules/indexer/code/git.go
+++ b/modules/indexer/code/git.go
@@ -129,8 +129,8 @@ func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revisio
changes.Updates = append(changes.Updates, updates...)
return nil
}
- lines := strings.Split(stdout, "\n")
- for _, line := range lines {
+ lines := strings.SplitSeq(stdout, "\n")
+ for line := range lines {
line = strings.TrimSpace(line)
if len(line) == 0 {
continue
diff --git a/modules/indexer/code/search.go b/modules/indexer/code/search.go
index e37aff8e59..a7a5d7d2e3 100644
--- a/modules/indexer/code/search.go
+++ b/modules/indexer/code/search.go
@@ -77,7 +77,7 @@ func HighlightSearchResultCode(filename, language string, lineNums []int, code s
// The lineNums outputted by highlight.Code might not match the original lineNums, because "highlight" removes the last `\n`
lines := make([]*ResultLine, min(len(highlightedLines), len(lineNums)))
- for i := 0; i < len(lines); i++ {
+ for i := range lines {
lines[i] = &ResultLine{
Num: lineNums[i],
FormattedContent: template.HTML(highlightedLines[i]),
diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go
index 6124ad2515..50951f9c88 100644
--- a/modules/indexer/issues/db/db.go
+++ b/modules/indexer/issues/db/db.go
@@ -6,6 +6,7 @@ package db
import (
"context"
"strings"
+ "sync"
"code.gitea.io/gitea/models/db"
issue_model "code.gitea.io/gitea/models/issues"
@@ -18,7 +19,7 @@ import (
"xorm.io/builder"
)
-var _ internal.Indexer = &Indexer{}
+var _ internal.Indexer = (*Indexer)(nil)
// Indexer implements Indexer interface to use database's like search
type Indexer struct {
@@ -29,11 +30,9 @@ func (i *Indexer) SupportedSearchModes() []indexer.SearchMode {
return indexer.SearchModesExactWords()
}
-func NewIndexer() *Indexer {
- return &Indexer{
- Indexer: &inner_db.Indexer{},
- }
-}
+var GetIndexer = sync.OnceValue(func() *Indexer {
+ return &Indexer{Indexer: &inner_db.Indexer{}}
+})
// Index dummy function
func (i *Indexer) Index(_ context.Context, _ ...*internal.IndexerData) error {
@@ -122,7 +121,11 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
}, nil
}
- ids, total, err := issue_model.IssueIDs(ctx, opt, cond)
+ return i.FindWithIssueOptions(ctx, opt, cond)
+}
+
+func (i *Indexer) FindWithIssueOptions(ctx context.Context, opt *issue_model.IssuesOptions, otherConds ...builder.Cond) (*internal.SearchResult, error) {
+ ids, total, err := issue_model.IssueIDs(ctx, opt, otherConds...)
if err != nil {
return nil, err
}
diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go
index 3b19d742ba..380a25dc23 100644
--- a/modules/indexer/issues/db/options.go
+++ b/modules/indexer/issues/db/options.go
@@ -6,6 +6,7 @@ package db
import (
"context"
"fmt"
+ "strings"
"code.gitea.io/gitea/models/db"
issue_model "code.gitea.io/gitea/models/issues"
@@ -34,7 +35,11 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m
case internal.SortByDeadlineAsc:
sortType = "nearduedate"
default:
- sortType = "newest"
+ if strings.HasPrefix(string(options.SortBy), issue_model.ScopeSortPrefix) {
+ sortType = string(options.SortBy)
+ } else {
+ sortType = "newest"
+ }
}
// See the comment of issues_model.SearchOptions for the reason why we need to convert
@@ -68,7 +73,6 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m
ExcludedLabelNames: nil,
IncludeMilestones: nil,
SortType: sortType,
- IssueIDs: nil,
UpdatedAfterUnix: options.UpdatedAfterUnix.Value(),
UpdatedBeforeUnix: options.UpdatedBeforeUnix.Value(),
PriorityRepoID: 0,
diff --git a/modules/indexer/issues/dboptions.go b/modules/indexer/issues/dboptions.go
index 4e2dff572a..f17724664d 100644
--- a/modules/indexer/issues/dboptions.go
+++ b/modules/indexer/issues/dboptions.go
@@ -4,12 +4,19 @@
package issues
import (
+ "strings"
+
"code.gitea.io/gitea/models/db"
issues_model "code.gitea.io/gitea/models/issues"
+ "code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/optional"
+ "code.gitea.io/gitea/modules/setting"
)
func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOptions {
+ if opts.IssueIDs != nil {
+ setting.PanicInDevOrTesting("Indexer SearchOptions doesn't support IssueIDs")
+ }
searchOpt := &SearchOptions{
Keyword: keyword,
RepoIDs: opts.RepoIDs,
@@ -95,7 +102,11 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp
// Unsupported sort type for search
fallthrough
default:
- searchOpt.SortBy = SortByUpdatedDesc
+ if strings.HasPrefix(opts.SortType, issues_model.ScopeSortPrefix) {
+ searchOpt.SortBy = internal.SortBy(opts.SortType)
+ } else {
+ searchOpt.SortBy = SortByUpdatedDesc
+ }
}
return searchOpt
diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go
index 4741235d47..8f25c84b76 100644
--- a/modules/indexer/issues/indexer.go
+++ b/modules/indexer/issues/indexer.go
@@ -103,7 +103,7 @@ func InitIssueIndexer(syncReindex bool) {
log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err)
}
case "db":
- issueIndexer = db.NewIndexer()
+ issueIndexer = db.GetIndexer()
case "meilisearch":
issueIndexer = meilisearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueConnAuth, setting.Indexer.IssueIndexerName)
existed, err = issueIndexer.Init(ctx)
@@ -217,7 +217,7 @@ func PopulateIssueIndexer(ctx context.Context) error {
return fmt.Errorf("shutdown before completion: %w", ctx.Err())
default:
}
- repos, _, err := repo_model.SearchRepositoryByName(ctx, &repo_model.SearchRepoOptions{
+ repos, _, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{
ListOptions: db_model.ListOptions{Page: page, PageSize: repo_model.RepositoryListDefaultPageSize},
OrderBy: db_model.SearchOrderByID,
Private: true,
@@ -291,19 +291,22 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, err
// So if the user creates an issue and list issues immediately, the issue may not be listed because the indexer needs time to index the issue.
// Even worse, the external indexer like elastic search may not be available for a while,
// and the user may not be able to list issues completely until it is available again.
- ix = db.NewIndexer()
+ ix = db.GetIndexer()
}
+
result, err := ix.Search(ctx, opts)
if err != nil {
return nil, 0, err
}
+ return SearchResultToIDSlice(result), result.Total, nil
+}
+func SearchResultToIDSlice(result *internal.SearchResult) []int64 {
ret := make([]int64, 0, len(result.Hits))
for _, hit := range result.Hits {
ret = append(ret, hit.ID)
}
-
- return ret, result.Total, nil
+ return ret
}
// CountIssues counts issues by options. It is a shortcut of SearchIssues(ctx, opts) but only returns the total count.
diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go
index a42ec9a2bc..7aebbbcd58 100644
--- a/modules/indexer/issues/internal/tests/tests.go
+++ b/modules/indexer/issues/internal/tests/tests.go
@@ -8,7 +8,6 @@
package tests
import (
- "context"
"fmt"
"slices"
"testing"
@@ -40,7 +39,7 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) {
data[v.ID] = v
}
require.NoError(t, indexer.Index(t.Context(), d...))
- require.NoError(t, waitData(indexer, int64(len(data))))
+ waitData(t, indexer, int64(len(data)))
}
defer func() {
@@ -54,13 +53,13 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) {
for _, v := range c.ExtraData {
data[v.ID] = v
}
- require.NoError(t, waitData(indexer, int64(len(data))))
+ waitData(t, indexer, int64(len(data)))
defer func() {
for _, v := range c.ExtraData {
require.NoError(t, indexer.Delete(t.Context(), v.ID))
delete(data, v.ID)
}
- require.NoError(t, waitData(indexer, int64(len(data))))
+ waitData(t, indexer, int64(len(data)))
}()
}
@@ -751,22 +750,10 @@ func countIndexerData(data map[int64]*internal.IndexerData, f func(v *internal.I
// waitData waits for the indexer to index all data.
// Some engines like Elasticsearch index data asynchronously, so we need to wait for a while.
-func waitData(indexer internal.Indexer, total int64) error {
- var actual int64
- for i := 0; i < 100; i++ {
- result, err := indexer.Search(context.Background(), &internal.SearchOptions{
- Paginator: &db.ListOptions{
- PageSize: 0,
- },
- })
- if err != nil {
- return err
- }
- actual = result.Total
- if actual == total {
- return nil
- }
- time.Sleep(100 * time.Millisecond)
- }
- return fmt.Errorf("waitData: expected %d, actual %d", total, actual)
+func waitData(t *testing.T, indexer internal.Indexer, total int64) {
+ assert.Eventually(t, func() bool {
+ result, err := indexer.Search(t.Context(), &internal.SearchOptions{Paginator: &db.ListOptions{}})
+ require.NoError(t, err)
+ return result.Total == total
+ }, 10*time.Second, 100*time.Millisecond, "expected total=%d", total)
}
diff --git a/modules/indexer/stats/db.go b/modules/indexer/stats/db.go
index 067a6f609b..199d493e97 100644
--- a/modules/indexer/stats/db.go
+++ b/modules/indexer/stats/db.go
@@ -8,6 +8,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/git/languagestats"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
@@ -62,7 +63,7 @@ func (db *DBIndexer) Index(id int64) error {
}
// Calculate and save language statistics to database
- stats, err := gitRepo.GetLanguageStats(commitID)
+ stats, err := languagestats.GetLanguageStats(gitRepo, commitID)
if err != nil {
if !setting.IsInTesting {
log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err)
generated by cgit v1.2.3 (git 2.39.1) at 2025-08-08 09:18:38 +0000