summaryrefslogtreecommitdiffstats
path: root/modules
diff options
context:
space:
mode:
Diffstat (limited to 'modules')
-rw-r--r--modules/context/repo.go1
-rw-r--r--modules/indexer/repo.go199
-rw-r--r--modules/search/search.go128
-rw-r--r--modules/setting/setting.go31
-rw-r--r--modules/util/path.go16
-rw-r--r--modules/util/util.go16
6 files changed, 378 insertions, 13 deletions
diff --git a/modules/context/repo.go b/modules/context/repo.go
index 3aaf1ce64a..704dc59f93 100644
--- a/modules/context/repo.go
+++ b/modules/context/repo.go
@@ -367,6 +367,7 @@ func RepoAssignment() macaron.Handler {
ctx.Data["DisableSSH"] = setting.SSH.Disabled
ctx.Data["ExposeAnonSSH"] = setting.SSH.ExposeAnonymous
ctx.Data["DisableHTTP"] = setting.Repository.DisableHTTPGit
+ ctx.Data["RepoSearchEnabled"] = setting.Indexer.RepoIndexerEnabled
ctx.Data["CloneLink"] = repo.CloneLink()
ctx.Data["WikiCloneLink"] = repo.WikiCloneLink()
diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go
new file mode 100644
index 0000000000..226e565e3e
--- /dev/null
+++ b/modules/indexer/repo.go
@@ -0,0 +1,199 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package indexer
+
+import (
+ "os"
+ "strings"
+
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/setting"
+
+ "github.com/blevesearch/bleve"
+ "github.com/blevesearch/bleve/analysis/analyzer/custom"
+ "github.com/blevesearch/bleve/analysis/token/camelcase"
+ "github.com/blevesearch/bleve/analysis/token/lowercase"
+ "github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+)
+
+const repoIndexerAnalyzer = "repoIndexerAnalyzer"
+
+// repoIndexer (thread-safe) index for repository contents
+var repoIndexer bleve.Index
+
+// RepoIndexerOp type of operation to perform on repo indexer
+type RepoIndexerOp int
+
+const (
+ // RepoIndexerOpUpdate add/update a file's contents
+ RepoIndexerOpUpdate = iota
+
+ // RepoIndexerOpDelete delete a file
+ RepoIndexerOpDelete
+)
+
+// RepoIndexerData data stored in the repo indexer
+type RepoIndexerData struct {
+ RepoID int64
+ Content string
+}
+
+// RepoIndexerUpdate an update to the repo indexer
+type RepoIndexerUpdate struct {
+ Filepath string
+ Op RepoIndexerOp
+ Data *RepoIndexerData
+}
+
+func (update RepoIndexerUpdate) addToBatch(batch *bleve.Batch) error {
+ id := filenameIndexerID(update.Data.RepoID, update.Filepath)
+ switch update.Op {
+ case RepoIndexerOpUpdate:
+ return batch.Index(id, update.Data)
+ case RepoIndexerOpDelete:
+ batch.Delete(id)
+ default:
+ log.Error(4, "Unrecognized repo indexer op: %d", update.Op)
+ }
+ return nil
+}
+
+// InitRepoIndexer initialize repo indexer
+func InitRepoIndexer(populateIndexer func() error) {
+ _, err := os.Stat(setting.Indexer.RepoPath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ if err = createRepoIndexer(); err != nil {
+ log.Fatal(4, "CreateRepoIndexer: %v", err)
+ }
+ if err = populateIndexer(); err != nil {
+ log.Fatal(4, "PopulateRepoIndex: %v", err)
+ }
+ } else {
+ log.Fatal(4, "InitRepoIndexer: %v", err)
+ }
+ } else {
+ repoIndexer, err = bleve.Open(setting.Indexer.RepoPath)
+ if err != nil {
+ log.Fatal(4, "InitRepoIndexer, open index: %v", err)
+ }
+ }
+}
+
+// createRepoIndexer create a repo indexer if one does not already exist
+func createRepoIndexer() error {
+ docMapping := bleve.NewDocumentMapping()
+ docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
+
+ textFieldMapping := bleve.NewTextFieldMapping()
+ docMapping.AddFieldMappingsAt("Content", textFieldMapping)
+
+ mapping := bleve.NewIndexMapping()
+ if err := addUnicodeNormalizeTokenFilter(mapping); err != nil {
+ return err
+ } else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]interface{}{
+ "type": custom.Name,
+ "char_filters": []string{},
+ "tokenizer": unicode.Name,
+ "token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
+ }); err != nil {
+ return err
+ }
+ mapping.DefaultAnalyzer = repoIndexerAnalyzer
+ mapping.AddDocumentMapping("repo", docMapping)
+ var err error
+ repoIndexer, err = bleve.New(setting.Indexer.RepoPath, mapping)
+ return err
+}
+
+func filenameIndexerID(repoID int64, filename string) string {
+ return indexerID(repoID) + "_" + filename
+}
+
+func filenameOfIndexerID(indexerID string) string {
+ index := strings.IndexByte(indexerID, '_')
+ if index == -1 {
+ log.Error(4, "Unexpected ID in repo indexer: %s", indexerID)
+ }
+ return indexerID[index+1:]
+}
+
+// RepoIndexerBatch batch to add updates to
+func RepoIndexerBatch() *Batch {
+ return &Batch{
+ batch: repoIndexer.NewBatch(),
+ index: repoIndexer,
+ }
+}
+
+// DeleteRepoFromIndexer delete all of a repo's files from indexer
+func DeleteRepoFromIndexer(repoID int64) error {
+ query := numericEqualityQuery(repoID, "RepoID")
+ searchRequest := bleve.NewSearchRequestOptions(query, 2147483647, 0, false)
+ result, err := repoIndexer.Search(searchRequest)
+ if err != nil {
+ return err
+ }
+ batch := RepoIndexerBatch()
+ for _, hit := range result.Hits {
+ batch.batch.Delete(hit.ID)
+ if err = batch.flushIfFull(); err != nil {
+ return err
+ }
+ }
+ return batch.Flush()
+}
+
+// RepoSearchResult result of performing a search in a repo
+type RepoSearchResult struct {
+ StartIndex int
+ EndIndex int
+ Filename string
+ Content string
+}
+
+// SearchRepoByKeyword searches for files in the specified repo.
+// Returns the matching file-paths
+func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
+ phraseQuery := bleve.NewMatchPhraseQuery(keyword)
+ phraseQuery.FieldVal = "Content"
+ phraseQuery.Analyzer = repoIndexerAnalyzer
+ indexerQuery := bleve.NewConjunctionQuery(
+ numericEqualityQuery(repoID, "RepoID"),
+ phraseQuery,
+ )
+ from := (page - 1) * pageSize
+ searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
+ searchRequest.Fields = []string{"Content"}
+ searchRequest.IncludeLocations = true
+
+ result, err := repoIndexer.Search(searchRequest)
+ if err != nil {
+ return 0, nil, err
+ }
+
+ searchResults := make([]*RepoSearchResult, len(result.Hits))
+ for i, hit := range result.Hits {
+ var startIndex, endIndex int = -1, -1
+ for _, locations := range hit.Locations["Content"] {
+ location := locations[0]
+ locationStart := int(location.Start)
+ locationEnd := int(location.End)
+ if startIndex < 0 || locationStart < startIndex {
+ startIndex = locationStart
+ }
+ if endIndex < 0 || locationEnd > endIndex {
+ endIndex = locationEnd
+ }
+ }
+ searchResults[i] = &RepoSearchResult{
+ StartIndex: startIndex,
+ EndIndex: endIndex,
+ Filename: filenameOfIndexerID(hit.ID),
+ Content: hit.Fields["Content"].(string),
+ }
+ }
+ return int64(result.Total), searchResults, nil
+}
diff --git a/modules/search/search.go b/modules/search/search.go
new file mode 100644
index 0000000000..1c13f11f8f
--- /dev/null
+++ b/modules/search/search.go
@@ -0,0 +1,128 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package search
+
+import (
+ "bytes"
+ gotemplate "html/template"
+ "strings"
+
+ "code.gitea.io/gitea/modules/highlight"
+ "code.gitea.io/gitea/modules/indexer"
+ "code.gitea.io/gitea/modules/util"
+)
+
+// Result a search result to display
+type Result struct {
+ Filename string
+ HighlightClass string
+ LineNumbers []int
+ FormattedLines gotemplate.HTML
+}
+
+func indices(content string, selectionStartIndex, selectionEndIndex int) (int, int) {
+ startIndex := selectionStartIndex
+ numLinesBefore := 0
+ for ; startIndex > 0; startIndex-- {
+ if content[startIndex-1] == '\n' {
+ if numLinesBefore == 1 {
+ break
+ }
+ numLinesBefore++
+ }
+ }
+
+ endIndex := selectionEndIndex
+ numLinesAfter := 0
+ for ; endIndex < len(content); endIndex++ {
+ if content[endIndex] == '\n' {
+ if numLinesAfter == 1 {
+ break
+ }
+ numLinesAfter++
+ }
+ }
+
+ return startIndex, endIndex
+}
+
+func writeStrings(buf *bytes.Buffer, strs ...string) error {
+ for _, s := range strs {
+ _, err := buf.WriteString(s)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*Result, error) {
+ startLineNum := 1 + strings.Count(result.Content[:startIndex], "\n")
+
+ var formattedLinesBuffer bytes.Buffer
+
+ contentLines := strings.SplitAfter(result.Content[startIndex:endIndex], "\n")
+ lineNumbers := make([]int, len(contentLines))
+ index := startIndex
+ for i, line := range contentLines {
+ var err error
+ if index < result.EndIndex &&
+ result.StartIndex < index+len(line) &&
+ result.StartIndex < result.EndIndex {
+ openActiveIndex := util.Max(result.StartIndex-index, 0)
+ closeActiveIndex := util.Min(result.EndIndex-index, len(line))
+ err = writeStrings(&formattedLinesBuffer,
+ `<li>`,
+ line[:openActiveIndex],
+ `<span class='active'>`,
+ line[openActiveIndex:closeActiveIndex],
+ `</span>`,
+ line[closeActiveIndex:],
+ `</li>`,
+ )
+ } else {
+ err = writeStrings(&formattedLinesBuffer,
+ `<li>`,
+ line,
+ `</li>`,
+ )
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ lineNumbers[i] = startLineNum + i
+ index += len(line)
+ }
+ return &Result{
+ Filename: result.Filename,
+ HighlightClass: highlight.FileNameToHighlightClass(result.Filename),
+ LineNumbers: lineNumbers,
+ FormattedLines: gotemplate.HTML(formattedLinesBuffer.String()),
+ }, nil
+}
+
+// PerformSearch perform a search on a repository
+func PerformSearch(repoID int64, keyword string, page, pageSize int) (int, []*Result, error) {
+ if len(keyword) == 0 {
+ return 0, nil, nil
+ }
+
+ total, results, err := indexer.SearchRepoByKeyword(repoID, keyword, page, pageSize)
+ if err != nil {
+ return 0, nil, err
+ }
+
+ displayResults := make([]*Result, len(results))
+
+ for i, result := range results {
+ startIndex, endIndex := indices(result.Content, result.StartIndex, result.EndIndex)
+ displayResults[i], err = searchResult(result, startIndex, endIndex)
+ if err != nil {
+ return 0, nil, err
+ }
+ }
+ return int(total), displayResults, nil
+}
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 6c89381f3b..a1106132df 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -140,8 +140,11 @@ var (
// Indexer settings
Indexer struct {
- IssuePath string
- UpdateQueueLength int
+ IssuePath string
+ RepoIndexerEnabled bool
+ RepoPath string
+ UpdateQueueLength int
+ MaxIndexerFileSize int64
}
// Webhook settings
@@ -234,12 +237,13 @@ var (
// UI settings
UI = struct {
- ExplorePagingNum int
- IssuePagingNum int
- FeedMaxCommitNum int
- ThemeColorMetaTag string
- MaxDisplayFileSize int64
- ShowUserEmail bool
+ ExplorePagingNum int
+ IssuePagingNum int
+ RepoSearchPagingNum int
+ FeedMaxCommitNum int
+ ThemeColorMetaTag string
+ MaxDisplayFileSize int64
+ ShowUserEmail bool
Admin struct {
UserPagingNum int
@@ -256,11 +260,12 @@ var (
Keywords string
} `ini:"ui.meta"`
}{
- ExplorePagingNum: 20,
- IssuePagingNum: 10,
- FeedMaxCommitNum: 5,
- ThemeColorMetaTag: `#6cc644`,
- MaxDisplayFileSize: 8388608,
+ ExplorePagingNum: 20,
+ IssuePagingNum: 10,
+ RepoSearchPagingNum: 10,
+ FeedMaxCommitNum: 5,
+ ThemeColorMetaTag: `#6cc644`,
+ MaxDisplayFileSize: 8388608,
Admin: struct {
UserPagingNum int
RepoPagingNum int
diff --git a/modules/util/path.go b/modules/util/path.go
new file mode 100644
index 0000000000..f79334209c
--- /dev/null
+++ b/modules/util/path.go
@@ -0,0 +1,16 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package util
+
+import "path/filepath"
+
+// EnsureAbsolutePath ensure that a path is absolute, making it
+// relative to absoluteBase if necessary
+func EnsureAbsolutePath(path string, absoluteBase string) string {
+ if filepath.IsAbs(path) {
+ return path
+ }
+ return filepath.Join(absoluteBase, path)
+}
diff --git a/modules/util/util.go b/modules/util/util.go
index 104c80f524..e99f951f21 100644
--- a/modules/util/util.go
+++ b/modules/util/util.go
@@ -38,3 +38,19 @@ func OptionalBoolOf(b bool) OptionalBool {
}
return OptionalBoolFalse
}
+
+// Max max of two ints
+func Max(a, b int) int {
+ if a < b {
+ return b
+ }
+ return a
+}
+
+// Min min of two ints
+func Min(a, b int) int {
+ if a > b {
+ return b
+ }
+ return a
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-28 08:37:04 +0000