Improve issue search (#2387)

* Improve issue indexer

* Fix new issue sqlite bug

* Different test indexer paths for each db

* Add integration indexer paths to make clean

1 files changed, 40 insertions, 124 deletions

diff --git a/models/issue_indexer.go b/models/issue_indexer.go
index 05b324f535..1e14268a0e 100644
--- a/models/issue_indexer.go
+++ b/models/issue_indexer.go
@@ -6,112 +6,21 @@ package models
 
 import (
 	"fmt"
-	"os"
-	"strconv"
-	"strings"
 
+	"code.gitea.io/gitea/modules/indexer"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
-
-	"github.com/blevesearch/bleve"
-	"github.com/blevesearch/bleve/analysis/analyzer/simple"
-	"github.com/blevesearch/bleve/search/query"
 )
 
-// issueIndexerUpdateQueue queue of issues that need to be updated in the issues
-// indexer
-var issueIndexerUpdateQueue chan *Issue
-
-// issueIndexer (thread-safe) index for searching issues
-var issueIndexer bleve.Index
-
-// issueIndexerData data stored in the issue indexer
-type issueIndexerData struct {
-	ID     int64
-	RepoID int64
-
-	Title   string
-	Content string
-}
-
-// numericQuery an numeric-equality query for the given value and field
-func numericQuery(value int64, field string) *query.NumericRangeQuery {
-	f := float64(value)
-	tru := true
-	q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru)
-	q.SetField(field)
-	return q
-}
-
-// SearchIssuesByKeyword searches for issues by given conditions.
-// Returns the matching issue IDs
-func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
-	terms := strings.Fields(strings.ToLower(keyword))
-	indexerQuery := bleve.NewConjunctionQuery(
-		numericQuery(repoID, "RepoID"),
-		bleve.NewDisjunctionQuery(
-			bleve.NewPhraseQuery(terms, "Title"),
-			bleve.NewPhraseQuery(terms, "Content"),
-		))
-	search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false)
-	search.Fields = []string{"ID"}
-
-	result, err := issueIndexer.Search(search)
-	if err != nil {
-		return nil, err
-	}
-
-	issueIDs := make([]int64, len(result.Hits))
-	for i, hit := range result.Hits {
-		issueIDs[i] = int64(hit.Fields["ID"].(float64))
-	}
-	return issueIDs, nil
-}
+// issueIndexerUpdateQueue queue of issue ids to be updated
+var issueIndexerUpdateQueue chan int64
 
 // InitIssueIndexer initialize issue indexer
 func InitIssueIndexer() {
-	_, err := os.Stat(setting.Indexer.IssuePath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			if err = createIssueIndexer(); err != nil {
-				log.Fatal(4, "CreateIssuesIndexer: %v", err)
-			}
-			if err = populateIssueIndexer(); err != nil {
-				log.Fatal(4, "PopulateIssuesIndex: %v", err)
-			}
-		} else {
-			log.Fatal(4, "InitIssuesIndexer: %v", err)
-		}
-	} else {
-		issueIndexer, err = bleve.Open(setting.Indexer.IssuePath)
-		if err != nil {
-			log.Fatal(4, "InitIssuesIndexer, open index: %v", err)
-		}
-	}
-	issueIndexerUpdateQueue = make(chan *Issue, setting.Indexer.UpdateQueueLength)
+	indexer.InitIssueIndexer(populateIssueIndexer)
+	issueIndexerUpdateQueue = make(chan int64, setting.Indexer.UpdateQueueLength)
 	go processIssueIndexerUpdateQueue()
-	// TODO close issueIndexer when Gitea closes
-}
-
-// createIssueIndexer create an issue indexer if one does not already exist
-func createIssueIndexer() error {
-	mapping := bleve.NewIndexMapping()
-	docMapping := bleve.NewDocumentMapping()
-
-	docMapping.AddFieldMappingsAt("ID", bleve.NewNumericFieldMapping())
-	docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
-
-	textFieldMapping := bleve.NewTextFieldMapping()
-	textFieldMapping.Analyzer = simple.Name
-	docMapping.AddFieldMappingsAt("Title", textFieldMapping)
-	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
-
-	mapping.AddDocumentMapping("issues", docMapping)
-
-	var err error
-	issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping)
-	return err
 }
 
 // populateIssueIndexer populate the issue indexer with issue data
@@ -127,57 +36,64 @@ func populateIssueIndexer() error {
 		if len(repos) == 0 {
 			return nil
 		}
-		batch := issueIndexer.NewBatch()
 		for _, repo := range repos {
 			issues, err := Issues(&IssuesOptions{
 				RepoID:   repo.ID,
 				IsClosed: util.OptionalBoolNone,
 				IsPull:   util.OptionalBoolNone,
 			})
-			if err != nil {
-				return fmt.Errorf("Issues: %v", err)
+			updates := make([]indexer.IssueIndexerUpdate, len(issues))
+			for i, issue := range issues {
+				updates[i] = issue.update()
 			}
-			for _, issue := range issues {
-				err = batch.Index(issue.indexUID(), issue.issueData())
-				if err != nil {
-					return fmt.Errorf("batch.Index: %v", err)
-				}
+			if err = indexer.BatchUpdateIssues(updates...); err != nil {
+				return fmt.Errorf("BatchUpdate: %v", err)
 			}
 		}
-		if err = issueIndexer.Batch(batch); err != nil {
-			return fmt.Errorf("index.Batch: %v", err)
-		}
 	}
 }
 
 func processIssueIndexerUpdateQueue() {
 	for {
 		select {
-		case issue := <-issueIndexerUpdateQueue:
-			if err := issueIndexer.Index(issue.indexUID(), issue.issueData()); err != nil {
+		case issueID := <-issueIndexerUpdateQueue:
+			issue, err := GetIssueByID(issueID)
+			if err != nil {
+				log.Error(4, "issuesIndexer.Index: %v", err)
+				continue
+			}
+			if err = indexer.UpdateIssue(issue.update()); err != nil {
 				log.Error(4, "issuesIndexer.Index: %v", err)
 			}
 		}
 	}
 }
 
-// indexUID a unique identifier for an issue used in full-text indices
-func (issue *Issue) indexUID() string {
-	return strconv.FormatInt(issue.ID, 36)
-}
-
-func (issue *Issue) issueData() *issueIndexerData {
-	return &issueIndexerData{
-		ID:      issue.ID,
-		RepoID:  issue.RepoID,
-		Title:   issue.Title,
-		Content: issue.Content,
+func (issue *Issue) update() indexer.IssueIndexerUpdate {
+	comments := make([]string, 0, 5)
+	for _, comment := range issue.Comments {
+		if comment.Type == CommentTypeComment {
+			comments = append(comments, comment.Content)
+		}
+	}
+	return indexer.IssueIndexerUpdate{
+		IssueID: issue.ID,
+		Data: &indexer.IssueIndexerData{
+			RepoID:   issue.RepoID,
+			Title:    issue.Title,
+			Content:  issue.Content,
+			Comments: comments,
+		},
 	}
 }
 
 // UpdateIssueIndexer add/update an issue to the issue indexer
-func UpdateIssueIndexer(issue *Issue) {
-	go func() {
-		issueIndexerUpdateQueue <- issue
-	}()
+func UpdateIssueIndexer(issueID int64) {
+	select {
+	case issueIndexerUpdateQueue <- issueID:
+	default:
+		go func() {
+			issueIndexerUpdateQueue <- issueID
+		}()
+	}
 }