diff options
Diffstat (limited to 'models/issue_indexer.go')
| -rw-r--r-- | models/issue_indexer.go | 164 |
1 files changed, 40 insertions, 124 deletions
diff --git a/models/issue_indexer.go b/models/issue_indexer.go index 05b324f535..1e14268a0e 100644 --- a/models/issue_indexer.go +++ b/models/issue_indexer.go @@ -6,112 +6,21 @@ package models import ( "fmt" - "os" - "strconv" - "strings" + "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/analysis/analyzer/simple" - "github.com/blevesearch/bleve/search/query" ) -// issueIndexerUpdateQueue queue of issues that need to be updated in the issues -// indexer -var issueIndexerUpdateQueue chan *Issue - -// issueIndexer (thread-safe) index for searching issues -var issueIndexer bleve.Index - -// issueIndexerData data stored in the issue indexer -type issueIndexerData struct { - ID int64 - RepoID int64 - - Title string - Content string -} - -// numericQuery an numeric-equality query for the given value and field -func numericQuery(value int64, field string) *query.NumericRangeQuery { - f := float64(value) - tru := true - q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru) - q.SetField(field) - return q -} - -// SearchIssuesByKeyword searches for issues by given conditions. -// Returns the matching issue IDs -func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) { - terms := strings.Fields(strings.ToLower(keyword)) - indexerQuery := bleve.NewConjunctionQuery( - numericQuery(repoID, "RepoID"), - bleve.NewDisjunctionQuery( - bleve.NewPhraseQuery(terms, "Title"), - bleve.NewPhraseQuery(terms, "Content"), - )) - search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false) - search.Fields = []string{"ID"} - - result, err := issueIndexer.Search(search) - if err != nil { - return nil, err - } - - issueIDs := make([]int64, len(result.Hits)) - for i, hit := range result.Hits { - issueIDs[i] = int64(hit.Fields["ID"].(float64)) - } - return issueIDs, nil -} +// issueIndexerUpdateQueue queue of issue ids to be updated +var issueIndexerUpdateQueue chan int64 // InitIssueIndexer initialize issue indexer func InitIssueIndexer() { - _, err := os.Stat(setting.Indexer.IssuePath) - if err != nil { - if os.IsNotExist(err) { - if err = createIssueIndexer(); err != nil { - log.Fatal(4, "CreateIssuesIndexer: %v", err) - } - if err = populateIssueIndexer(); err != nil { - log.Fatal(4, "PopulateIssuesIndex: %v", err) - } - } else { - log.Fatal(4, "InitIssuesIndexer: %v", err) - } - } else { - issueIndexer, err = bleve.Open(setting.Indexer.IssuePath) - if err != nil { - log.Fatal(4, "InitIssuesIndexer, open index: %v", err) - } - } - issueIndexerUpdateQueue = make(chan *Issue, setting.Indexer.UpdateQueueLength) + indexer.InitIssueIndexer(populateIssueIndexer) + issueIndexerUpdateQueue = make(chan int64, setting.Indexer.UpdateQueueLength) go processIssueIndexerUpdateQueue() - // TODO close issueIndexer when Gitea closes -} - -// createIssueIndexer create an issue indexer if one does not already exist -func createIssueIndexer() error { - mapping := bleve.NewIndexMapping() - docMapping := bleve.NewDocumentMapping() - - docMapping.AddFieldMappingsAt("ID", bleve.NewNumericFieldMapping()) - docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping()) - - textFieldMapping := bleve.NewTextFieldMapping() - textFieldMapping.Analyzer = simple.Name - docMapping.AddFieldMappingsAt("Title", textFieldMapping) - docMapping.AddFieldMappingsAt("Content", textFieldMapping) - - mapping.AddDocumentMapping("issues", docMapping) - - var err error - issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping) - return err } // populateIssueIndexer populate the issue indexer with issue data @@ -127,57 +36,64 @@ func populateIssueIndexer() error { if len(repos) == 0 { return nil } - batch := issueIndexer.NewBatch() for _, repo := range repos { issues, err := Issues(&IssuesOptions{ RepoID: repo.ID, IsClosed: util.OptionalBoolNone, IsPull: util.OptionalBoolNone, }) - if err != nil { - return fmt.Errorf("Issues: %v", err) + updates := make([]indexer.IssueIndexerUpdate, len(issues)) + for i, issue := range issues { + updates[i] = issue.update() } - for _, issue := range issues { - err = batch.Index(issue.indexUID(), issue.issueData()) - if err != nil { - return fmt.Errorf("batch.Index: %v", err) - } + if err = indexer.BatchUpdateIssues(updates...); err != nil { + return fmt.Errorf("BatchUpdate: %v", err) } } - if err = issueIndexer.Batch(batch); err != nil { - return fmt.Errorf("index.Batch: %v", err) - } } } func processIssueIndexerUpdateQueue() { for { select { - case issue := <-issueIndexerUpdateQueue: - if err := issueIndexer.Index(issue.indexUID(), issue.issueData()); err != nil { + case issueID := <-issueIndexerUpdateQueue: + issue, err := GetIssueByID(issueID) + if err != nil { + log.Error(4, "issuesIndexer.Index: %v", err) + continue + } + if err = indexer.UpdateIssue(issue.update()); err != nil { log.Error(4, "issuesIndexer.Index: %v", err) } } } } -// indexUID a unique identifier for an issue used in full-text indices -func (issue *Issue) indexUID() string { - return strconv.FormatInt(issue.ID, 36) -} - -func (issue *Issue) issueData() *issueIndexerData { - return &issueIndexerData{ - ID: issue.ID, - RepoID: issue.RepoID, - Title: issue.Title, - Content: issue.Content, +func (issue *Issue) update() indexer.IssueIndexerUpdate { + comments := make([]string, 0, 5) + for _, comment := range issue.Comments { + if comment.Type == CommentTypeComment { + comments = append(comments, comment.Content) + } + } + return indexer.IssueIndexerUpdate{ + IssueID: issue.ID, + Data: &indexer.IssueIndexerData{ + RepoID: issue.RepoID, + Title: issue.Title, + Content: issue.Content, + Comments: comments, + }, } } // UpdateIssueIndexer add/update an issue to the issue indexer -func UpdateIssueIndexer(issue *Issue) { - go func() { - issueIndexerUpdateQueue <- issue - }() +func UpdateIssueIndexer(issueID int64) { + select { + case issueIndexerUpdateQueue <- issueID: + default: + go func() { + issueIndexerUpdateQueue <- issueID + }() + } } |