* add more tests and docs for issue indexer, add db indexer type for searching from database
* fix typo
* fix typo
* fix lint
* improve docs
; Backoff time per DB retry (time.Duration)
DB_RETRY_BACKOFF = 3s
-
[indexer]
+; Issue indexer type, currently support: bleve or db, default is bleve
+ISSUE_INDEXER_TYPE = bleve
+; Issue indexer storage path, available when ISSUE_INDEXER_TYPE is bleve
ISSUE_INDEXER_PATH = indexers/issues.bleve
+; Issue indexer queue, currently support: channel or levelqueue, default is levelqueue
+ISSUE_INDEXER_QUEUE_TYPE = levelqueue
+; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the queue will be saved path,
+; default is indexers/issues.queue
+ISSUE_INDEXER_QUEUE_DIR = indexers/issues.queue
+; Batch queue number, default is 20
+ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20
+
; repo indexer by default disabled, since it uses a lot of disk space
REPO_INDEXER_ENABLED = false
REPO_INDEXER_PATH = indexers/repos.bleve
## Indexer (`indexer`)
+- `ISSUE_INDEXER_TYPE`: **bleve**: Issue indexer type, currently support: bleve or db, if it's db, below issue indexer item will be invalid.
- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search.
+- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently support: channel or levelqueue
+- `ISSUE_INDEXER_QUEUE_DIR`: **indexers/issues.queue**: When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the queue will be saved path
+- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number
+
- `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space).
- `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
- `PATH`: Tidb 或者 SQLite3 数据文件存放路径。
- `LOG_SQL`: **true**: 显示生成的SQL，默认为真。
+
+## Indexer (`indexer`)
+
+- `ISSUE_INDEXER_TYPE`: **bleve**: 工单索引类型，当前支持 `bleve` 或 `db`，当为 `db` 时其它工单索引项可不用设置。
+- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: 工单索引文件存放路径，当索引类型为 `bleve` 时有效。
+- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: 工单索引队列类型，当前支持 `channel` 或 `levelqueue`。
+- `ISSUE_INDEXER_QUEUE_DIR`: **indexers/issues.queue**: 当 `ISSUE_INDEXER_QUEUE_TYPE` 为 `levelqueue` 时，保存索引队列的磁盘路径。
+- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: 队列处理中批量提交数量。
+
+- `REPO_INDEXER_ENABLED`: **false**: 是否启用代码搜索（启用后会占用比较大的磁盘空间）。
+- `REPO_INDEXER_PATH`: **indexers/repos.bleve**: 用于代码搜索的索引文件路径。
+- `UPDATE_BUFFER_LEN`: **20**: 代码索引请求的缓冲区长度。
+- `MAX_FILE_SIZE`: **1048576**: 进行解析的源代码文件的最大长度，小于该值时才会索引。
+
## Security (`security`)
- `INSTALL_LOCK`: 是否允许运行安装向导，(跟管理员账号有关，十分重要)。
return openResult, closedResult
}
+// SearchIssueIDsByKeyword search issues on database
+func SearchIssueIDsByKeyword(kw string, repoID int64, limit, start int) (int64, []int64, error) {
+ var repoCond = builder.Eq{"repo_id": repoID}
+ var subQuery = builder.Select("id").From("issue").Where(repoCond)
+ var cond = builder.And(
+ repoCond,
+ builder.Or(
+ builder.Like{"name", kw},
+ builder.Like{"content", kw},
+ builder.In("id", builder.Select("issue_id").
+ From("comment").
+ Where(builder.And(
+ builder.Eq{"type": CommentTypeComment},
+ builder.In("issue_id", subQuery),
+ builder.Like{"content", kw},
+ )),
+ ),
+ ),
+ )
+
+ var ids = make([]int64, 0, limit)
+ err := x.Distinct("id").Table("issue").Where(cond).Limit(limit, start).Find(&ids)
+ if err != nil {
+ return 0, nil, err
+ }
+
+ total, err := x.Distinct("id").Table("issue").Where(cond).Count()
+ if err != nil {
+ return 0, nil, err
+ }
+
+ return total, ids, nil
+}
+
func updateIssue(e Engine, issue *Issue) error {
_, err := e.ID(issue.ID).AllCols().Update(issue)
if err != nil {
assert.NoError(t, ms.loadTotalTimes(x))
assert.Equal(t, int64(3662), ms.TotalTrackedTime)
}
+
+func TestIssue_SearchIssueIDsByKeyword(t *testing.T) {
+ assert.NoError(t, PrepareTestDatabase())
+
+ total, ids, err := SearchIssueIDsByKeyword("issue2", 1, 10, 0)
+ assert.NoError(t, err)
+ assert.EqualValues(t, 1, total)
+ assert.EqualValues(t, []int64{2}, ids)
+
+ total, ids, err = SearchIssueIDsByKeyword("first", 1, 10, 0)
+ assert.NoError(t, err)
+ assert.EqualValues(t, 1, total)
+ assert.EqualValues(t, []int64{1}, ids)
+
+ total, ids, err = SearchIssueIDsByKeyword("for", 1, 10, 0)
+ assert.NoError(t, err)
+ assert.EqualValues(t, 4, total)
+ assert.EqualValues(t, []int64{1, 2, 3, 5}, ids)
+
+ // issue1's comment id 2
+ total, ids, err = SearchIssueIDsByKeyword("good", 1, 10, 0)
+ assert.NoError(t, err)
+ assert.EqualValues(t, 1, total)
+ assert.EqualValues(t, []int64{1}, ids)
+}
--- /dev/null
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package issues
+
+import "code.gitea.io/gitea/models"
+
+// DBIndexer implements Indexer inteface to use database's like search
+type DBIndexer struct {
+}
+
+// Init dummy function
+func (db *DBIndexer) Init() (bool, error) {
+ return false, nil
+}
+
+// Index dummy function
+func (db *DBIndexer) Index(issue []*IndexerData) error {
+ return nil
+}
+
+// Delete dummy function
+func (db *DBIndexer) Delete(ids ...int64) error {
+ return nil
+}
+
+// Search dummy function
+func (db *DBIndexer) Search(kw string, repoID int64, limit, start int) (*SearchResult, error) {
+ total, ids, err := models.SearchIssueIDsByKeyword(kw, repoID, limit, start)
+ if err != nil {
+ return nil, err
+ }
+ var result = SearchResult{
+ Total: total,
+ Hits: make([]Match, 0, limit),
+ }
+ for _, id := range ids {
+ result.Hits = append(result.Hits, Match{
+ ID: id,
+ RepoID: repoID,
+ })
+ }
+ return &result, nil
+}
// SearchResult represents search results
type SearchResult struct {
- Hits []Match
+ Total int64
+ Hits []Match
}
// Indexer defines an inteface to indexer issues contents
// all issue index done.
func InitIssueIndexer(syncReindex bool) error {
var populate bool
+ var dummyQueue bool
switch setting.Indexer.IssueType {
case "bleve":
issueIndexer = NewBleveIndexer(setting.Indexer.IssuePath)
return err
}
populate = !exist
+ case "db":
+ issueIndexer = &DBIndexer{}
+ dummyQueue = true
default:
return fmt.Errorf("unknow issue indexer type: %s", setting.Indexer.IssueType)
}
+ if dummyQueue {
+ return nil
+ }
+
var err error
switch setting.Indexer.IssueIndexerQueueType {
case setting.LevelQueueType:
ids, err = SearchIssuesByKeyword(1, "for")
assert.NoError(t, err)
assert.EqualValues(t, []int64{1, 2, 3, 5}, ids)
+
+ ids, err = SearchIssuesByKeyword(1, "good")
+ assert.NoError(t, err)
+ assert.EqualValues(t, []int64{1}, ids)
}
// Queue defines an interface to save an issue indexer queue
type Queue interface {
Run() error
- Push(*IndexerData)
+ Push(*IndexerData) error
+}
+
+// DummyQueue represents an empty queue
+type DummyQueue struct {
+}
+
+// Run starts to run the queue
+func (b *DummyQueue) Run() error {
+ return nil
+}
+
+// Push pushes data to indexer
+func (b *DummyQueue) Push(*IndexerData) error {
+ return nil
}
for {
select {
case data := <-c.queue:
+ if data.IsDelete {
+ c.indexer.Delete(data.IDs...)
+ continue
+ }
+
datas = append(datas, data)
if len(datas) >= c.batchNumber {
c.indexer.Index(datas)
}
// Push will push the indexer data to queue
-func (c *ChannelQueue) Push(data *IndexerData) {
+func (c *ChannelQueue) Push(data *IndexerData) error {
c.queue <- data
+ return nil
}
}
// Push will push the indexer data to queue
-func (l *LevelQueue) Push(data *IndexerData) {
+func (l *LevelQueue) Push(data *IndexerData) error {
bs, err := json.Marshal(data)
if err != nil {
- log.Error(4, "Marshal: %v", err)
- return
- }
- err = l.queue.LPush(bs)
- if err != nil {
- log.Error(4, "LPush: %v", err)
+ return err
}
+ return l.queue.LPush(bs)
}
func newIndexerService() {
sec := Cfg.Section("indexer")
+ Indexer.IssueType = sec.Key("ISSUE_INDEXER_TYPE").MustString("bleve")
Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString(path.Join(AppDataPath, "indexers/issues.bleve"))
if !filepath.IsAbs(Indexer.IssuePath) {
Indexer.IssuePath = path.Join(AppWorkPath, Indexer.IssuePath)
projects / gitea.git / commitdiff