Automatically pause queue if index service is unavailable (#15066)

* Handle keyword search error when issue indexer service is not available

* Implement automatic disabling and resume of code indexer queue

4 files changed, 33 insertions, 15 deletions

diff --git a/routers/web/explore/code.go b/routers/web/explore/code.go
index d2acefde92..640a5a0e4f 100644
--- a/routers/web/explore/code.go
+++ b/routers/web/explore/code.go
@@ -87,17 +87,27 @@ func Code(ctx *context.Context) {
 
 		ctx.Data["RepoMaps"] = rightRepoMap
 
-		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
+		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
 		if err != nil {
-			ctx.ServerError("SearchResults", err)
-			return
+			if code_indexer.IsAvailable() {
+				ctx.ServerError("SearchResults", err)
+				return
+			}
+			ctx.Data["CodeIndexerUnavailable"] = true
+		} else {
+			ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
 		}
 		// if non-login user or isAdmin, no need to check UnitTypeCode
 	} else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
-		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
+		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(ctx, repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
 		if err != nil {
-			ctx.ServerError("SearchResults", err)
-			return
+			if code_indexer.IsAvailable() {
+				ctx.ServerError("SearchResults", err)
+				return
+			}
+			ctx.Data["CodeIndexerUnavailable"] = true
+		} else {
+			ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
 		}
 
 		loadRepoIDs := make([]int64, 0, len(searchResults))
diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go
index c4928054a0..952135eaf5 100644
--- a/routers/web/repo/issue.go
+++ b/routers/web/repo/issue.go
@@ -161,10 +161,13 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti
 
 	var issueIDs []int64
 	if len(keyword) > 0 {
-		issueIDs, err = issue_indexer.SearchIssuesByKeyword([]int64{repo.ID}, keyword)
+		issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword)
 		if err != nil {
-			ctx.ServerError("issueIndexer.Search", err)
-			return
+			if issue_indexer.IsAvailable() {
+				ctx.ServerError("issueIndexer.Search", err)
+				return
+			}
+			ctx.Data["IssueIndexerUnavailable"] = true
 		}
 		if len(issueIDs) == 0 {
 			forceEmpty = true
diff --git a/routers/web/repo/search.go b/routers/web/repo/search.go
index 67539c3d7e..e33fe38dea 100644
--- a/routers/web/repo/search.go
+++ b/routers/web/repo/search.go
@@ -30,11 +30,16 @@ func Search(ctx *context.Context) {
 	queryType := ctx.FormTrim("t")
 	isMatch := queryType == "match"
 
-	total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch([]int64{ctx.Repo.Repository.ID},
+	total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch(ctx, []int64{ctx.Repo.Repository.ID},
 		language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
 	if err != nil {
-		ctx.ServerError("SearchResults", err)
-		return
+		if code_indexer.IsAvailable() {
+			ctx.ServerError("SearchResults", err)
+			return
+		}
+		ctx.Data["CodeIndexerUnavailable"] = true
+	} else {
+		ctx.Data["CodeIndexerUnavailable"] = !code_indexer.IsAvailable()
 	}
 	ctx.Data["Keyword"] = keyword
 	ctx.Data["Language"] = language
diff --git a/routers/web/user/home.go b/routers/web/user/home.go
index 13fa9bd8c4..33512d97c0 100644
--- a/routers/web/user/home.go
+++ b/routers/web/user/home.go
@@ -438,7 +438,7 @@ func buildIssueOverview(ctx *context.Context, unitType unit.Type) {
 
 	// Execute keyword search for issues.
 	// USING NON-FINAL STATE OF opts FOR A QUERY.
-	issueIDsFromSearch, err := issueIDsFromSearch(ctxUser, keyword, opts)
+	issueIDsFromSearch, err := issueIDsFromSearch(ctx, ctxUser, keyword, opts)
 	if err != nil {
 		ctx.ServerError("issueIDsFromSearch", err)
 		return
@@ -673,7 +673,7 @@ func getRepoIDs(reposQuery string) []int64 {
 	return repoIDs
 }
 
-func issueIDsFromSearch(ctxUser *user_model.User, keyword string, opts *models.IssuesOptions) ([]int64, error) {
+func issueIDsFromSearch(ctx *context.Context, ctxUser *user_model.User, keyword string, opts *models.IssuesOptions) ([]int64, error) {
 	if len(keyword) == 0 {
 		return []int64{}, nil
 	}
@@ -682,7 +682,7 @@ func issueIDsFromSearch(ctxUser *user_model.User, keyword string, opts *models.I
 	if err != nil {
 		return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %v", err)
 	}
-	issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(searchRepoIDs, keyword)
+	issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword)
 	if err != nil {
 		return nil, fmt.Errorf("SearchIssuesByKeyword: %v", err)
 	}