Refactor and enhance issue indexer to support both searching, filtering and paging (#26012)

Fix #24662.

Replace #24822 and #25708 (although it has been merged)


## Background

In the past, Gitea supported issue searching with a keyword and
conditions in a less efficient way. It worked by searching for issues
with the keyword and obtaining limited IDs (as it is heavy to get all)
on the indexer (bleve/elasticsearch/meilisearch), and then querying with
conditions on the database to find a subset of the found IDs. This is
why the results could be incomplete.

To solve this issue, we need to store all fields that could be used as
conditions in the indexer and support both keyword and additional
conditions when searching with the indexer.

## Major changes

- Redefine `IndexerData` to include all fields that could be used as
filter conditions.
- Refactor `Search(ctx context.Context, kw string, repoIDs []int64,
limit, start int, state string)` to `Search(ctx context.Context, options
*SearchOptions)`, so it supports more conditions now.
- Change the data type stored in `issueIndexerQueue`. Use
`IndexerMetadata` instead of `IndexerData` in case the data has been
updated while it is in the queue. This also reduces the storage size of
the queue.
- Enhance searching with Bleve/Elasticsearch/Meilisearch, make them
fully support `SearchOptions`. Also, update the data versions.
- Keep most logic of database indexer, but remove
`issues.SearchIssueIDsByKeyword` in `models` to avoid confusion where is
the entry point to search issues.
- Start a Meilisearch instance to test it in unit tests.
- Add unit tests with almost full coverage to test
Bleve/Elasticsearch/Meilisearch indexer.

---------

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

2 files changed, 81 insertions, 50 deletions

diff --git a/routers/web/user/home.go b/routers/web/user/home.go
index 5f1e0eb427..77974a84a2 100644
--- a/routers/web/user/home.go
+++ b/routers/web/user/home.go
@@ -22,6 +22,7 @@ import (
 	"code.gitea.io/gitea/models/unit"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/container"
 	"code.gitea.io/gitea/modules/context"
 	issue_indexer "code.gitea.io/gitea/modules/indexer/issues"
 	"code.gitea.io/gitea/modules/json"
@@ -466,21 +467,16 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
 	keyword := strings.Trim(ctx.FormString("q"), " ")
 	ctx.Data["Keyword"] = keyword
 
-	// Execute keyword search for issues.
-	// USING NON-FINAL STATE OF opts FOR A QUERY.
-	issueIDsFromSearch, err := issueIDsFromSearch(ctx, ctxUser, keyword, opts)
-	if err != nil {
-		ctx.ServerError("issueIDsFromSearch", err)
-		return
-	}
-
-	// Ensure no issues are returned if a keyword was provided that didn't match any issues.
-	var forceEmpty bool
-
-	if len(issueIDsFromSearch) > 0 {
-		opts.IssueIDs = issueIDsFromSearch
-	} else if len(keyword) > 0 {
-		forceEmpty = true
+	accessibleRepos := container.Set[int64]{}
+	{
+		ids, err := issues_model.GetRepoIDsForIssuesOptions(opts, ctxUser)
+		if err != nil {
+			ctx.ServerError("GetRepoIDsForIssuesOptions", err)
+			return
+		}
+		for _, id := range ids {
+			accessibleRepos.Add(id)
+		}
 	}
 
 	// Educated guess: Do or don't show closed issues.
@@ -490,12 +486,21 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
 	// Filter repos and count issues in them. Count will be used later.
 	// USING NON-FINAL STATE OF opts FOR A QUERY.
 	var issueCountByRepo map[int64]int64
-	if !forceEmpty {
-		issueCountByRepo, err = issues_model.CountIssuesByRepo(ctx, opts)
+	{
+		issueIDs, err := issueIDsFromSearch(ctx, keyword, opts)
 		if err != nil {
-			ctx.ServerError("CountIssuesByRepo", err)
+			ctx.ServerError("issueIDsFromSearch", err)
 			return
 		}
+		if len(issueIDs) > 0 { // else, no issues found, just leave issueCountByRepo empty
+			opts.IssueIDs = issueIDs
+			issueCountByRepo, err = issues_model.CountIssuesByRepo(ctx, opts)
+			if err != nil {
+				ctx.ServerError("CountIssuesByRepo", err)
+				return
+			}
+			opts.IssueIDs = nil // reset, the opts will be used later
+		}
 	}
 
 	// Make sure page number is at least 1. Will be posted to ctx.Data.
@@ -503,14 +508,17 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
 	if page <= 1 {
 		page = 1
 	}
-	opts.Page = page
-	opts.PageSize = setting.UI.IssuePagingNum
+	opts.Paginator = &db.ListOptions{
+		Page:     page,
+		PageSize: setting.UI.IssuePagingNum,
+	}
 
 	// Get IDs for labels (a filter option for issues/pulls).
 	// Required for IssuesOptions.
 	var labelIDs []int64
 	selectedLabels := ctx.FormString("labels")
 	if len(selectedLabels) > 0 && selectedLabels != "0" {
+		var err error
 		labelIDs, err = base.StringsToInt64s(strings.Split(selectedLabels, ","))
 		if err != nil {
 			ctx.ServerError("StringsToInt64s", err)
@@ -521,7 +529,16 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
 
 	// Parse ctx.FormString("repos") and remember matched repo IDs for later.
 	// Gets set when clicking filters on the issues overview page.
-	opts.RepoIDs = getRepoIDs(ctx.FormString("repos"))
+	repoIDs := getRepoIDs(ctx.FormString("repos"))
+	if len(repoIDs) == 0 {
+		repoIDs = accessibleRepos.Values()
+	} else {
+		// Remove repo IDs that are not accessible to the user.
+		repoIDs = util.SliceRemoveAllFunc(repoIDs, func(v int64) bool {
+			return !accessibleRepos.Contains(v)
+		})
+	}
+	opts.RepoIDs = repoIDs
 
 	// ------------------------------
 	// Get issues as defined by opts.
@@ -529,15 +546,18 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
 
 	// Slice of Issues that will be displayed on the overview page
 	// USING FINAL STATE OF opts FOR A QUERY.
-	var issues []*issues_model.Issue
-	if !forceEmpty {
-		issues, err = issues_model.Issues(ctx, opts)
+	var issues issues_model.IssueList
+	{
+		issueIDs, err := issueIDsFromSearch(ctx, keyword, opts)
 		if err != nil {
-			ctx.ServerError("Issues", err)
+			ctx.ServerError("issueIDsFromSearch", err)
+			return
+		}
+		issues, err = issues_model.GetIssuesByIDs(ctx, issueIDs, true)
+		if err != nil {
+			ctx.ServerError("GetIssuesByIDs", err)
 			return
 		}
-	} else {
-		issues = []*issues_model.Issue{}
 	}
 
 	// ----------------------------------
@@ -576,12 +596,12 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
 	// Fill stats to post to ctx.Data.
 	// -------------------------------
 	var issueStats *issues_model.IssueStats
-	if !forceEmpty {
+	{
 		statsOpts := issues_model.IssuesOptions{
 			User:       ctx.Doer,
 			IsPull:     util.OptionalBoolOf(isPullList),
 			IsClosed:   util.OptionalBoolOf(isShowClosed),
-			IssueIDs:   issueIDsFromSearch,
+			IssueIDs:   nil,
 			IsArchived: util.OptionalBoolFalse,
 			LabelIDs:   opts.LabelIDs,
 			Org:        org,
@@ -589,13 +609,29 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
 			RepoCond:   opts.RepoCond,
 		}
 
-		issueStats, err = issues_model.GetUserIssueStats(filterMode, statsOpts)
-		if err != nil {
-			ctx.ServerError("GetUserIssueStats Shown", err)
-			return
+		if keyword != "" {
+			statsOpts.RepoIDs = opts.RepoIDs
+			allIssueIDs, err := issueIDsFromSearch(ctx, keyword, &statsOpts)
+			if err != nil {
+				ctx.ServerError("issueIDsFromSearch", err)
+				return
+			}
+			statsOpts.IssueIDs = allIssueIDs
+		}
+
+		if keyword != "" && len(statsOpts.IssueIDs) == 0 {
+			// So it did search with the keyword, but no issue found.
+			// Just set issueStats to empty.
+			issueStats = &issues_model.IssueStats{}
+		} else {
+			// So it did search with the keyword, and found some issues. It needs to get issueStats of these issues.
+			// Or the keyword is empty, so it doesn't need issueIDs as filter, just get issueStats with statsOpts.
+			issueStats, err = issues_model.GetUserIssueStats(filterMode, statsOpts)
+			if err != nil {
+				ctx.ServerError("GetUserIssueStats", err)
+				return
+			}
 		}
-	} else {
-		issueStats = &issues_model.IssueStats{}
 	}
 
 	// Will be posted to ctx.Data.
@@ -629,9 +665,13 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
 
 	ctx.Data["IssueRefEndNames"], ctx.Data["IssueRefURLs"] = issue_service.GetRefEndNamesAndURLs(issues, ctx.FormString("RepoLink"))
 
+	if err := issues.LoadAttributes(ctx); err != nil {
+		ctx.ServerError("issues.LoadAttributes", err)
+		return
+	}
 	ctx.Data["Issues"] = issues
 
-	approvalCounts, err := issues_model.IssueList(issues).GetApprovalCounts(ctx)
+	approvalCounts, err := issues.GetApprovalCounts(ctx)
 	if err != nil {
 		ctx.ServerError("ApprovalCounts", err)
 		return
@@ -716,21 +756,12 @@ func getRepoIDs(reposQuery string) []int64 {
 	return repoIDs
 }
 
-func issueIDsFromSearch(ctx *context.Context, ctxUser *user_model.User, keyword string, opts *issues_model.IssuesOptions) ([]int64, error) {
-	if len(keyword) == 0 {
-		return []int64{}, nil
-	}
-
-	searchRepoIDs, err := issues_model.GetRepoIDsForIssuesOptions(opts, ctxUser)
+func issueIDsFromSearch(ctx *context.Context, keyword string, opts *issues_model.IssuesOptions) ([]int64, error) {
+	ids, _, err := issue_indexer.SearchIssues(ctx, issue_indexer.ToSearchOptions(keyword, opts))
 	if err != nil {
-		return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %w", err)
+		return nil, fmt.Errorf("SearchIssues: %w", err)
 	}
-	issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword, ctx.FormString("state"))
-	if err != nil {
-		return nil, fmt.Errorf("SearchIssuesByKeyword: %w", err)
-	}
-
-	return issueIDsFromSearch, nil
+	return ids, nil
 }
 
 func loadRepoByIDs(ctxUser *user_model.User, issueCountByRepo map[int64]int64, unitType unit.Type) (map[int64]*repo_model.Repository, error) {
diff --git a/routers/web/user/notification.go b/routers/web/user/notification.go
index cae12f4126..60ae628445 100644
--- a/routers/web/user/notification.go
+++ b/routers/web/user/notification.go
@@ -263,7 +263,7 @@ func NotificationSubscriptions(ctx *context.Context) {
 		return
 	}
 	issues, err := issues_model.Issues(ctx, &issues_model.IssuesOptions{
-		ListOptions: db.ListOptions{
+		Paginator: &db.ListOptions{
 			PageSize: setting.UI.IssuePagingNum,
 			Page:     page,
 		},