diff options
Diffstat (limited to 'modules/indexer/issues')
| -rw-r--r-- | modules/indexer/issues/bleve/bleve.go | 50 | ||||
| -rw-r--r-- | modules/indexer/issues/db/db.go | 50 | ||||
| -rw-r--r-- | modules/indexer/issues/db/options.go | 14 | ||||
| -rw-r--r-- | modules/indexer/issues/dboptions.go | 19 | ||||
| -rw-r--r-- | modules/indexer/issues/elasticsearch/elasticsearch.go | 44 | ||||
| -rw-r--r-- | modules/indexer/issues/indexer.go | 25 | ||||
| -rw-r--r-- | modules/indexer/issues/indexer_test.go | 64 | ||||
| -rw-r--r-- | modules/indexer/issues/internal/indexer.go | 14 | ||||
| -rw-r--r-- | modules/indexer/issues/internal/model.go | 8 | ||||
| -rw-r--r-- | modules/indexer/issues/internal/tests/tests.go | 86 | ||||
| -rw-r--r-- | modules/indexer/issues/meilisearch/meilisearch.go | 26 | ||||
| -rw-r--r-- | modules/indexer/issues/meilisearch/meilisearch_test.go | 12 | ||||
| -rw-r--r-- | modules/indexer/issues/util.go | 7 |
13 files changed, 280 insertions, 139 deletions
diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index bf51bd6c14..39d96cab98 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -5,10 +5,14 @@ package bleve import ( "context" + "strconv" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/util" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" @@ -120,6 +124,10 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWordsFuzzy() +} + // NewIndexer creates a new bleve local indexer func NewIndexer(indexDir string) *Indexer { inner := inner_bleve.NewIndexer(indexDir, issueIndexerLatestVersion, generateIssueIndexMapping) @@ -157,16 +165,24 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( var queries []query.Query if options.Keyword != "" { - fuzziness := 0 - if options.IsFuzzyKeyword { - fuzziness = inner_bleve.GuessFuzzinessByKeyword(options.Keyword) + searchMode := util.IfZero(options.SearchMode, b.SupportedSearchModes()[0].ModeValue) + if searchMode == indexer.SearchModeWords || searchMode == indexer.SearchModeFuzzy { + fuzziness := 0 + if searchMode == indexer.SearchModeFuzzy { + fuzziness = inner_bleve.GuessFuzzinessByKeyword(options.Keyword) + } + queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ + inner_bleve.MatchAndQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness), + inner_bleve.MatchAndQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness), + inner_bleve.MatchAndQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness), + }...)) + } else /* exact */ { + queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ + inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, 0), + inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, 0), + inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, 0), + }...)) } - - queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{ - inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness), - inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness), - inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness), - }...)) } if len(options.RepoIDs) > 0 || options.AllPublic { @@ -232,12 +248,20 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( queries = append(queries, inner_bleve.NumericEqualityQuery(options.ProjectColumnID.Value(), "project_board_id")) } - if options.PosterID.Has() { - queries = append(queries, inner_bleve.NumericEqualityQuery(options.PosterID.Value(), "poster_id")) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + queries = append(queries, inner_bleve.NumericEqualityQuery(posterIDInt64, "poster_id")) } - if options.AssigneeID.Has() { - queries = append(queries, inner_bleve.NumericEqualityQuery(options.AssigneeID.Value(), "assignee_id")) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + queries = append(queries, inner_bleve.NumericRangeInclusiveQuery(optional.Some[int64](1), optional.None[int64](), "assignee_id")) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + queries = append(queries, inner_bleve.NumericEqualityQuery(assigneeIDInt64, "assignee_id")) + } } if options.MentionID.Has() { diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 6c9cfcf670..50951f9c88 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -5,29 +5,35 @@ package db import ( "context" + "strings" + "sync" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_db "code.gitea.io/gitea/modules/indexer/internal/db" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/util" "xorm.io/builder" ) -var _ internal.Indexer = &Indexer{} +var _ internal.Indexer = (*Indexer)(nil) // Indexer implements Indexer interface to use database's like search type Indexer struct { indexer_internal.Indexer } -func NewIndexer() *Indexer { - return &Indexer{ - Indexer: &inner_db.Indexer{}, - } +func (i *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWords() } +var GetIndexer = sync.OnceValue(func() *Indexer { + return &Indexer{Indexer: &inner_db.Indexer{}} +}) + // Index dummy function func (i *Indexer) Index(_ context.Context, _ ...*internal.IndexerData) error { return nil @@ -38,6 +44,26 @@ func (i *Indexer) Delete(_ context.Context, _ ...int64) error { return nil } +func buildMatchQuery(mode indexer.SearchModeType, colName, keyword string) builder.Cond { + if mode == indexer.SearchModeExact { + return db.BuildCaseInsensitiveLike(colName, keyword) + } + + // match words + cond := builder.NewCond() + fields := strings.Fields(keyword) + if len(fields) == 0 { + return builder.Expr("1=1") + } + for _, field := range fields { + if field == "" { + continue + } + cond = cond.And(db.BuildCaseInsensitiveLike(colName, field)) + } + return cond +} + // Search searches for issues func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (*internal.SearchResult, error) { // FIXME: I tried to avoid importing models here, but it seems to be impossible. @@ -58,16 +84,16 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( repoCond = builder.Eq{"repo_id": options.RepoIDs[0]} } subQuery := builder.Select("id").From("issue").Where(repoCond) - + searchMode := util.IfZero(options.SearchMode, i.SupportedSearchModes()[0].ModeValue) cond = builder.Or( - db.BuildCaseInsensitiveLike("issue.name", options.Keyword), - db.BuildCaseInsensitiveLike("issue.content", options.Keyword), + buildMatchQuery(searchMode, "issue.name", options.Keyword), + buildMatchQuery(searchMode, "issue.content", options.Keyword), builder.In("issue.id", builder.Select("issue_id"). From("comment"). Where(builder.And( builder.Eq{"type": issue_model.CommentTypeComment}, builder.In("issue_id", subQuery), - db.BuildCaseInsensitiveLike("content", options.Keyword), + buildMatchQuery(searchMode, "content", options.Keyword), )), ), ) @@ -95,7 +121,11 @@ func (i *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( }, nil } - ids, total, err := issue_model.IssueIDs(ctx, opt, cond) + return i.FindWithIssueOptions(ctx, opt, cond) +} + +func (i *Indexer) FindWithIssueOptions(ctx context.Context, opt *issue_model.IssuesOptions, otherConds ...builder.Cond) (*internal.SearchResult, error) { + ids, total, err := issue_model.IssueIDs(ctx, opt, otherConds...) if err != nil { return nil, err } diff --git a/modules/indexer/issues/db/options.go b/modules/indexer/issues/db/options.go index 42834f6e88..380a25dc23 100644 --- a/modules/indexer/issues/db/options.go +++ b/modules/indexer/issues/db/options.go @@ -6,6 +6,7 @@ package db import ( "context" "fmt" + "strings" "code.gitea.io/gitea/models/db" issue_model "code.gitea.io/gitea/models/issues" @@ -34,7 +35,11 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m case internal.SortByDeadlineAsc: sortType = "nearduedate" default: - sortType = "newest" + if strings.HasPrefix(string(options.SortBy), issue_model.ScopeSortPrefix) { + sortType = string(options.SortBy) + } else { + sortType = "newest" + } } // See the comment of issues_model.SearchOptions for the reason why we need to convert @@ -54,7 +59,7 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m RepoIDs: options.RepoIDs, AllPublic: options.AllPublic, RepoCond: nil, - AssigneeID: optional.Some(convertID(options.AssigneeID)), + AssigneeID: options.AssigneeID, PosterID: options.PosterID, MentionedID: convertID(options.MentionID), ReviewRequestedID: convertID(options.ReviewRequestedID), @@ -68,14 +73,13 @@ func ToDBOptions(ctx context.Context, options *internal.SearchOptions) (*issue_m ExcludedLabelNames: nil, IncludeMilestones: nil, SortType: sortType, - IssueIDs: nil, UpdatedAfterUnix: options.UpdatedAfterUnix.Value(), UpdatedBeforeUnix: options.UpdatedBeforeUnix.Value(), PriorityRepoID: 0, IsArchived: options.IsArchived, - Org: nil, + Owner: nil, Team: nil, - User: nil, + Doer: nil, } if len(options.MilestoneIDs) == 1 && options.MilestoneIDs[0] == 0 { diff --git a/modules/indexer/issues/dboptions.go b/modules/indexer/issues/dboptions.go index 4f6ad96d22..f17724664d 100644 --- a/modules/indexer/issues/dboptions.go +++ b/modules/indexer/issues/dboptions.go @@ -4,12 +4,19 @@ package issues import ( + "strings" + "code.gitea.io/gitea/models/db" issues_model "code.gitea.io/gitea/models/issues" + "code.gitea.io/gitea/modules/indexer/issues/internal" "code.gitea.io/gitea/modules/optional" + "code.gitea.io/gitea/modules/setting" ) func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOptions { + if opts.IssueIDs != nil { + setting.PanicInDevOrTesting("Indexer SearchOptions doesn't support IssueIDs") + } searchOpt := &SearchOptions{ Keyword: keyword, RepoIDs: opts.RepoIDs, @@ -45,11 +52,7 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp searchOpt.ProjectID = optional.Some[int64](0) // Those issues with no project(projectid==0) } - if opts.AssigneeID.Value() == db.NoConditionID { - searchOpt.AssigneeID = optional.Some[int64](0) // FIXME: this is inconsistent from other places, 0 means "no assignee" - } else if opts.AssigneeID.Value() != 0 { - searchOpt.AssigneeID = opts.AssigneeID - } + searchOpt.AssigneeID = opts.AssigneeID // See the comment of issues_model.SearchOptions for the reason why we need to convert convertID := func(id int64) optional.Option[int64] { @@ -99,7 +102,11 @@ func ToSearchOptions(keyword string, opts *issues_model.IssuesOptions) *SearchOp // Unsupported sort type for search fallthrough default: - searchOpt.SortBy = SortByUpdatedDesc + if strings.HasPrefix(opts.SortType, issues_model.ScopeSortPrefix) { + searchOpt.SortBy = internal.SortBy(opts.SortType) + } else { + searchOpt.SortBy = SortByUpdatedDesc + } } return searchOpt diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 4c293f3f2a..9d627466ef 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -5,14 +5,15 @@ package elasticsearch import ( "context" - "fmt" "strconv" "strings" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch" "code.gitea.io/gitea/modules/indexer/issues/internal" + "code.gitea.io/gitea/modules/util" "github.com/olivere/elastic/v7" ) @@ -33,6 +34,11 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + // TODO: es supports fuzzy search, but our code doesn't at the moment, and actually the default fuzziness is already "AUTO" + return indexer.SearchModesExactWords() +} + // NewIndexer creates a new elasticsearch indexer func NewIndexer(url, indexerName string) *Indexer { inner := inner_elasticsearch.NewIndexer(url, indexerName, issueIndexerLatestVersion, defaultMapping) @@ -89,7 +95,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er issue := issues[0] _, err := b.inner.Client.Index(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", issue.ID)). + Id(strconv.FormatInt(issue.ID, 10)). BodyJson(issue). Do(ctx) return err @@ -100,7 +106,7 @@ func (b *Indexer) Index(ctx context.Context, issues ...*internal.IndexerData) er reqs = append(reqs, elastic.NewBulkIndexRequest(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", issue.ID)). + Id(strconv.FormatInt(issue.ID, 10)). Doc(issue), ) } @@ -119,7 +125,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { } else if len(ids) == 1 { _, err := b.inner.Client.Delete(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", ids[0])). + Id(strconv.FormatInt(ids[0], 10)). Do(ctx) return err } @@ -129,7 +135,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { reqs = append(reqs, elastic.NewBulkDeleteRequest(). Index(b.inner.VersionedIndexName()). - Id(fmt.Sprintf("%d", id)), + Id(strconv.FormatInt(id, 10)), ) } @@ -146,12 +152,12 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query := elastic.NewBoolQuery() if options.Keyword != "" { - searchType := esMultiMatchTypePhrasePrefix - if options.IsFuzzyKeyword { - searchType = esMultiMatchTypeBestFields + searchMode := util.IfZero(options.SearchMode, b.SupportedSearchModes()[0].ModeValue) + if searchMode == indexer.SearchModeExact { + query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(esMultiMatchTypePhrasePrefix)) + } else /* words */ { + query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(esMultiMatchTypeBestFields).Operator("and")) } - - query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(searchType)) } if len(options.RepoIDs) > 0 { @@ -205,12 +211,22 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query.Must(elastic.NewTermQuery("project_board_id", options.ProjectColumnID.Value())) } - if options.PosterID.Has() { - query.Must(elastic.NewTermQuery("poster_id", options.PosterID.Value())) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + query.Must(elastic.NewTermQuery("poster_id", posterIDInt64)) } - if options.AssigneeID.Has() { - query.Must(elastic.NewTermQuery("assignee_id", options.AssigneeID.Value())) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + q := elastic.NewRangeQuery("assignee_id") + q.Gte(1) + query.Must(q) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + query.Must(elastic.NewTermQuery("assignee_id", assigneeIDInt64)) + } } if options.MentionID.Has() { diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index c82dc0867e..8f25c84b76 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -14,6 +14,7 @@ import ( db_model "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/issues/bleve" "code.gitea.io/gitea/modules/indexer/issues/db" "code.gitea.io/gitea/modules/indexer/issues/elasticsearch" @@ -102,7 +103,7 @@ func InitIssueIndexer(syncReindex bool) { log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err) } case "db": - issueIndexer = db.NewIndexer() + issueIndexer = db.GetIndexer() case "meilisearch": issueIndexer = meilisearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueConnAuth, setting.Indexer.IssueIndexerName) existed, err = issueIndexer.Init(ctx) @@ -216,7 +217,7 @@ func PopulateIssueIndexer(ctx context.Context) error { return fmt.Errorf("shutdown before completion: %w", ctx.Err()) default: } - repos, _, err := repo_model.SearchRepositoryByName(ctx, &repo_model.SearchRepoOptions{ + repos, _, err := repo_model.SearchRepositoryByName(ctx, repo_model.SearchRepoOptions{ ListOptions: db_model.ListOptions{Page: page, PageSize: repo_model.RepositoryListDefaultPageSize}, OrderBy: db_model.SearchOrderByID, Private: true, @@ -281,7 +282,7 @@ const ( // SearchIssues search issues by options. func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, error) { - indexer := *globalIndexer.Load() + ix := *globalIndexer.Load() if opts.Keyword == "" || opts.IsKeywordNumeric() { // This is a conservative shortcut. @@ -290,20 +291,22 @@ func SearchIssues(ctx context.Context, opts *SearchOptions) ([]int64, int64, err // So if the user creates an issue and list issues immediately, the issue may not be listed because the indexer needs time to index the issue. // Even worse, the external indexer like elastic search may not be available for a while, // and the user may not be able to list issues completely until it is available again. - indexer = db.NewIndexer() + ix = db.GetIndexer() } - result, err := indexer.Search(ctx, opts) + result, err := ix.Search(ctx, opts) if err != nil { return nil, 0, err } + return SearchResultToIDSlice(result), result.Total, nil +} +func SearchResultToIDSlice(result *internal.SearchResult) []int64 { ret := make([]int64, 0, len(result.Hits)) for _, hit := range result.Hits { ret = append(ret, hit.ID) } - - return ret, result.Total, nil + return ret } // CountIssues counts issues by options. It is a shortcut of SearchIssues(ctx, opts) but only returns the total count. @@ -313,3 +316,11 @@ func CountIssues(ctx context.Context, opts *SearchOptions) (int64, error) { _, total, err := SearchIssues(ctx, opts) return total, err } + +func SupportedSearchModes() []indexer.SearchMode { + gi := globalIndexer.Load() + if gi == nil { + return nil + } + return (*gi).SupportedSearchModes() +} diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go index 8043d33eeb..3e38ac49b7 100644 --- a/modules/indexer/issues/indexer_test.go +++ b/modules/indexer/issues/indexer_test.go @@ -4,7 +4,6 @@ package issues import ( - "context" "testing" "code.gitea.io/gitea/models/db" @@ -45,6 +44,7 @@ func TestDBSearchIssues(t *testing.T) { t.Run("search issues with order", searchIssueWithOrder) t.Run("search issues in project", searchIssueInProject) t.Run("search issues with paginator", searchIssueWithPaginator) + t.Run("search issues with any assignee", searchIssueWithAnyAssignee) } func searchIssueWithKeyword(t *testing.T) { @@ -83,9 +83,11 @@ func searchIssueWithKeyword(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) - require.NoError(t, err) - assert.Equal(t, test.expectedIDs, issueIDs) + t.Run(test.opts.Keyword, func(t *testing.T) { + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) + assert.Equal(t, test.expectedIDs, issueIDs) + }) } } @@ -118,7 +120,7 @@ func searchIssueByIndex(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -162,7 +164,7 @@ func searchIssueInRepo(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -175,19 +177,19 @@ func searchIssueByID(t *testing.T) { }{ { opts: SearchOptions{ - PosterID: optional.Some(int64(1)), + PosterID: "1", }, expectedIDs: []int64{11, 6, 3, 2, 1}, }, { opts: SearchOptions{ - AssigneeID: optional.Some(int64(1)), + AssigneeID: "1", }, expectedIDs: []int64{6, 1}, }, { - // NOTE: This tests no assignees filtering and also ToSearchOptions() to ensure it will set AssigneeID to 0 when it is passed as -1. - opts: *ToSearchOptions("", &issues.IssuesOptions{AssigneeID: optional.Some(db.NoConditionID)}), + // NOTE: This tests no assignees filtering and also ToSearchOptions() to ensure it handles the filter correctly + opts: *ToSearchOptions("", &issues.IssuesOptions{AssigneeID: "(none)"}), expectedIDs: []int64{22, 21, 16, 15, 14, 13, 12, 11, 20, 5, 19, 18, 10, 7, 4, 9, 8, 3, 2}, }, { @@ -232,7 +234,7 @@ func searchIssueByID(t *testing.T) { } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -257,7 +259,7 @@ func searchIssueIsPull(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -282,7 +284,7 @@ func searchIssueIsClosed(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -307,7 +309,7 @@ func searchIssueIsArchived(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -332,7 +334,7 @@ func searchIssueByMilestoneID(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -363,7 +365,7 @@ func searchIssueByLabelID(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -382,7 +384,7 @@ func searchIssueByTime(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -401,7 +403,7 @@ func searchIssueWithOrder(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -432,7 +434,7 @@ func searchIssueInProject(t *testing.T) { }, } for _, test := range tests { - issueIDs, _, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, _, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) } @@ -455,7 +457,29 @@ func searchIssueWithPaginator(t *testing.T) { }, } for _, test := range tests { - issueIDs, total, err := SearchIssues(context.TODO(), &test.opts) + issueIDs, total, err := SearchIssues(t.Context(), &test.opts) + require.NoError(t, err) + assert.Equal(t, test.expectedIDs, issueIDs) + assert.Equal(t, test.expectedTotal, total) + } +} + +func searchIssueWithAnyAssignee(t *testing.T) { + tests := []struct { + opts SearchOptions + expectedIDs []int64 + expectedTotal int64 + }{ + { + SearchOptions{ + AssigneeID: "(any)", + }, + []int64{17, 6, 1}, + 3, + }, + } + for _, test := range tests { + issueIDs, total, err := SearchIssues(t.Context(), &test.opts) require.NoError(t, err) assert.Equal(t, test.expectedIDs, issueIDs) assert.Equal(t, test.expectedTotal, total) diff --git a/modules/indexer/issues/internal/indexer.go b/modules/indexer/issues/internal/indexer.go index 95740bc598..59c6f48485 100644 --- a/modules/indexer/issues/internal/indexer.go +++ b/modules/indexer/issues/internal/indexer.go @@ -5,8 +5,9 @@ package internal import ( "context" - "fmt" + "errors" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/internal" ) @@ -16,6 +17,7 @@ type Indexer interface { Index(ctx context.Context, issue ...*IndexerData) error Delete(ctx context.Context, ids ...int64) error Search(ctx context.Context, options *SearchOptions) (*SearchResult, error) + SupportedSearchModes() []indexer.SearchMode } // NewDummyIndexer returns a dummy indexer @@ -29,14 +31,18 @@ type dummyIndexer struct { internal.Indexer } +func (d *dummyIndexer) SupportedSearchModes() []indexer.SearchMode { + return nil +} + func (d *dummyIndexer) Index(_ context.Context, _ ...*IndexerData) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Delete(_ context.Context, _ ...int64) error { - return fmt.Errorf("indexer is not ready") + return errors.New("indexer is not ready") } func (d *dummyIndexer) Search(_ context.Context, _ *SearchOptions) (*SearchResult, error) { - return nil, fmt.Errorf("indexer is not ready") + return nil, errors.New("indexer is not ready") } diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 09dcbf4804..0d4f0f727d 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -7,6 +7,7 @@ import ( "strconv" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/timeutil" ) @@ -77,7 +78,7 @@ type SearchResult struct { type SearchOptions struct { Keyword string // keyword to search - IsFuzzyKeyword bool // if false the levenshtein distance is 0 + SearchMode indexer.SearchModeType RepoIDs []int64 // repository IDs which the issues belong to AllPublic bool // if include all public repositories @@ -96,9 +97,8 @@ type SearchOptions struct { ProjectID optional.Option[int64] // project the issues belong to ProjectColumnID optional.Option[int64] // project column the issues belong to - PosterID optional.Option[int64] // poster of the issues - - AssigneeID optional.Option[int64] // assignee of the issues, zero means no assignee + PosterID string // poster of the issues, "(none)" or "(any)" or a user ID + AssigneeID string // assignee of the issues, "(none)" or "(any)" or a user ID MentionID optional.Option[int64] // mentioned user of the issues diff --git a/modules/indexer/issues/internal/tests/tests.go b/modules/indexer/issues/internal/tests/tests.go index 94ce8520bf..7aebbbcd58 100644 --- a/modules/indexer/issues/internal/tests/tests.go +++ b/modules/indexer/issues/internal/tests/tests.go @@ -8,7 +8,6 @@ package tests import ( - "context" "fmt" "slices" "testing" @@ -24,10 +23,10 @@ import ( ) func TestIndexer(t *testing.T, indexer internal.Indexer) { - _, err := indexer.Init(context.Background()) + _, err := indexer.Init(t.Context()) require.NoError(t, err) - require.NoError(t, indexer.Ping(context.Background())) + require.NoError(t, indexer.Ping(t.Context())) var ( ids []int64 @@ -39,32 +38,32 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { ids = append(ids, v.ID) data[v.ID] = v } - require.NoError(t, indexer.Index(context.Background(), d...)) - require.NoError(t, waitData(indexer, int64(len(data)))) + require.NoError(t, indexer.Index(t.Context(), d...)) + waitData(t, indexer, int64(len(data))) } defer func() { - require.NoError(t, indexer.Delete(context.Background(), ids...)) + require.NoError(t, indexer.Delete(t.Context(), ids...)) }() for _, c := range cases { t.Run(c.Name, func(t *testing.T) { if len(c.ExtraData) > 0 { - require.NoError(t, indexer.Index(context.Background(), c.ExtraData...)) + require.NoError(t, indexer.Index(t.Context(), c.ExtraData...)) for _, v := range c.ExtraData { data[v.ID] = v } - require.NoError(t, waitData(indexer, int64(len(data)))) + waitData(t, indexer, int64(len(data))) defer func() { for _, v := range c.ExtraData { - require.NoError(t, indexer.Delete(context.Background(), v.ID)) + require.NoError(t, indexer.Delete(t.Context(), v.ID)) delete(data, v.ID) } - require.NoError(t, waitData(indexer, int64(len(data)))) + waitData(t, indexer, int64(len(data))) }() } - result, err := indexer.Search(context.Background(), c.SearchOptions) + result, err := indexer.Search(t.Context(), c.SearchOptions) require.NoError(t, err) if c.Expected != nil { @@ -80,7 +79,7 @@ func TestIndexer(t *testing.T, indexer internal.Indexer) { // test counting c.SearchOptions.Paginator = &db.ListOptions{PageSize: 0} - countResult, err := indexer.Search(context.Background(), c.SearchOptions) + countResult, err := indexer.Search(t.Context(), c.SearchOptions) require.NoError(t, err) assert.Empty(t, countResult.Hits) assert.Equal(t, result.Total, countResult.Total) @@ -93,7 +92,7 @@ var cases = []*testIndexerCase{ Name: "default", SearchOptions: &internal.SearchOptions{}, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) }, }, @@ -379,7 +378,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - PosterID: optional.Some(int64(1)), + PosterID: "1", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -397,7 +396,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - AssigneeID: optional.Some(int64(1)), + AssigneeID: "1", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -415,7 +414,7 @@ var cases = []*testIndexerCase{ Paginator: &db.ListOptions{ PageSize: 5, }, - AssigneeID: optional.Some(int64(0)), + AssigneeID: "(none)", }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { assert.Len(t, result.Hits, 5) @@ -526,7 +525,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCreatedDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -542,7 +541,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByUpdatedDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -558,7 +557,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCommentsDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -574,7 +573,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByDeadlineDesc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -590,7 +589,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCreatedAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -606,7 +605,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByUpdatedAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -622,7 +621,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByCommentsAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -638,7 +637,7 @@ var cases = []*testIndexerCase{ SortBy: internal.SortByDeadlineAsc, }, Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { - assert.Equal(t, len(data), len(result.Hits)) + assert.Len(t, result.Hits, len(data)) assert.Equal(t, len(data), int(result.Total)) for i, v := range result.Hits { if i < len(result.Hits)-1 { @@ -647,6 +646,21 @@ var cases = []*testIndexerCase{ } }, }, + { + Name: "SearchAnyAssignee", + SearchOptions: &internal.SearchOptions{ + AssigneeID: "(any)", + }, + Expected: func(t *testing.T, data map[int64]*internal.IndexerData, result *internal.SearchResult) { + assert.Len(t, result.Hits, 180) + for _, v := range result.Hits { + assert.GreaterOrEqual(t, data[v.ID].AssigneeID, int64(1)) + } + assert.Equal(t, countIndexerData(data, func(v *internal.IndexerData) bool { + return v.AssigneeID >= 1 + }), result.Total) + }, + }, } type testIndexerCase struct { @@ -736,22 +750,10 @@ func countIndexerData(data map[int64]*internal.IndexerData, f func(v *internal.I // waitData waits for the indexer to index all data. // Some engines like Elasticsearch index data asynchronously, so we need to wait for a while. -func waitData(indexer internal.Indexer, total int64) error { - var actual int64 - for i := 0; i < 100; i++ { - result, err := indexer.Search(context.Background(), &internal.SearchOptions{ - Paginator: &db.ListOptions{ - PageSize: 0, - }, - }) - if err != nil { - return err - } - actual = result.Total - if actual == total { - return nil - } - time.Sleep(100 * time.Millisecond) - } - return fmt.Errorf("waitData: expected %d, actual %d", total, actual) +func waitData(t *testing.T, indexer internal.Indexer, total int64) { + assert.Eventually(t, func() bool { + result, err := indexer.Search(t.Context(), &internal.SearchOptions{Paginator: &db.ListOptions{}}) + require.NoError(t, err) + return result.Total == total + }, 10*time.Second, 100*time.Millisecond, "expected total=%d", total) } diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 1066e96272..759a98473f 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "code.gitea.io/gitea/modules/indexer" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_meilisearch "code.gitea.io/gitea/modules/indexer/internal/meilisearch" "code.gitea.io/gitea/modules/indexer/issues/internal" @@ -35,6 +36,10 @@ type Indexer struct { indexer_internal.Indexer // do not composite inner_meilisearch.Indexer directly to avoid exposing too much } +func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { + return indexer.SearchModesExactWords() +} + // NewIndexer creates a new meilisearch indexer func NewIndexer(url, apiKey, indexerName string) *Indexer { settings := &meilisearch.Settings{ @@ -182,12 +187,20 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( query.And(inner_meilisearch.NewFilterEq("project_board_id", options.ProjectColumnID.Value())) } - if options.PosterID.Has() { - query.And(inner_meilisearch.NewFilterEq("poster_id", options.PosterID.Value())) + if options.PosterID != "" { + // "(none)" becomes 0, it means no poster + posterIDInt64, _ := strconv.ParseInt(options.PosterID, 10, 64) + query.And(inner_meilisearch.NewFilterEq("poster_id", posterIDInt64)) } - if options.AssigneeID.Has() { - query.And(inner_meilisearch.NewFilterEq("assignee_id", options.AssigneeID.Value())) + if options.AssigneeID != "" { + if options.AssigneeID == "(any)" { + query.And(inner_meilisearch.NewFilterGte("assignee_id", 1)) + } else { + // "(none)" becomes 0, it means no assignee + assigneeIDInt64, _ := strconv.ParseInt(options.AssigneeID, 10, 64) + query.And(inner_meilisearch.NewFilterEq("assignee_id", assigneeIDInt64)) + } } if options.MentionID.Has() { @@ -230,9 +243,8 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) ( limit = 1 } - keyword := options.Keyword - if !options.IsFuzzyKeyword { - // to make it non fuzzy ("typo tolerance" in meilisearch terms), we have to quote the keyword(s) + keyword := options.Keyword // default to match "words" + if options.SearchMode == indexer.SearchModeExact { // https://www.meilisearch.com/docs/reference/api/search#phrase-search keyword = doubleQuoteKeyword(keyword) } diff --git a/modules/indexer/issues/meilisearch/meilisearch_test.go b/modules/indexer/issues/meilisearch/meilisearch_test.go index a3a332554a..2fea4004cb 100644 --- a/modules/indexer/issues/meilisearch/meilisearch_test.go +++ b/modules/indexer/issues/meilisearch/meilisearch_test.go @@ -74,13 +74,13 @@ func TestConvertHits(t *testing.T) { } hits, err := convertHits(validResponse) assert.NoError(t, err) - assert.EqualValues(t, []internal.Match{{ID: 11}, {ID: 22}, {ID: 33}}, hits) + assert.Equal(t, []internal.Match{{ID: 11}, {ID: 22}, {ID: 33}}, hits) } func TestDoubleQuoteKeyword(t *testing.T) { - assert.EqualValues(t, "", doubleQuoteKeyword("")) - assert.EqualValues(t, `"a" "b" "c"`, doubleQuoteKeyword("a b c")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) - assert.EqualValues(t, `"a" "d" "g"`, doubleQuoteKeyword(`a "" "d" """g`)) + assert.Empty(t, doubleQuoteKeyword("")) + assert.Equal(t, `"a" "b" "c"`, doubleQuoteKeyword("a b c")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword("a d g")) + assert.Equal(t, `"a" "d" "g"`, doubleQuoteKeyword(`a "" "d" """g`)) } diff --git a/modules/indexer/issues/util.go b/modules/indexer/issues/util.go index deb19adc49..19d835a1d8 100644 --- a/modules/indexer/issues/util.go +++ b/modules/indexer/issues/util.go @@ -92,6 +92,11 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD projectID = issue.Project.ID } + projectColumnID, err := issue.ProjectColumnID(ctx) + if err != nil { + return nil, false, err + } + return &internal.IndexerData{ ID: issue.ID, RepoID: issue.RepoID, @@ -106,7 +111,7 @@ func getIssueIndexerData(ctx context.Context, issueID int64) (*internal.IndexerD NoLabel: len(labels) == 0, MilestoneID: issue.MilestoneID, ProjectID: projectID, - ProjectColumnID: issue.ProjectColumnID(ctx), + ProjectColumnID: projectColumnID, PosterID: issue.PosterID, AssigneeID: issue.AssigneeID, MentionIDs: mentionIDs, |