* add more tests and docs for issue indexer, add db indexer type for searching from database * fix typo * fix typo * fix lint * improve docstags/v1.9.0-dev
@@ -253,9 +253,19 @@ DB_RETRIES = 10 | |||
; Backoff time per DB retry (time.Duration) | |||
DB_RETRY_BACKOFF = 3s | |||
[indexer] | |||
; Issue indexer type, currently support: bleve or db, default is bleve | |||
ISSUE_INDEXER_TYPE = bleve | |||
; Issue indexer storage path, available when ISSUE_INDEXER_TYPE is bleve | |||
ISSUE_INDEXER_PATH = indexers/issues.bleve | |||
; Issue indexer queue, currently support: channel or levelqueue, default is levelqueue | |||
ISSUE_INDEXER_QUEUE_TYPE = levelqueue | |||
; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the queue will be saved path, | |||
; default is indexers/issues.queue | |||
ISSUE_INDEXER_QUEUE_DIR = indexers/issues.queue | |||
; Batch queue number, default is 20 | |||
ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20 | |||
; repo indexer by default disabled, since it uses a lot of disk space | |||
REPO_INDEXER_ENABLED = false | |||
REPO_INDEXER_PATH = indexers/repos.bleve |
@@ -154,7 +154,12 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. | |||
## Indexer (`indexer`) | |||
- `ISSUE_INDEXER_TYPE`: **bleve**: Issue indexer type, currently support: bleve or db, if it's db, below issue indexer item will be invalid. | |||
- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search. | |||
- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently support: channel or levelqueue | |||
- `ISSUE_INDEXER_QUEUE_DIR`: **indexers/issues.queue**: When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the queue will be saved path | |||
- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number | |||
- `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space). | |||
- `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. | |||
- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. |
@@ -82,6 +82,20 @@ menu: | |||
- `PATH`: Tidb 或者 SQLite3 数据文件存放路径。 | |||
- `LOG_SQL`: **true**: 显示生成的SQL,默认为真。 | |||
## Indexer (`indexer`) | |||
- `ISSUE_INDEXER_TYPE`: **bleve**: 工单索引类型,当前支持 `bleve` 或 `db`,当为 `db` 时其它工单索引项可不用设置。 | |||
- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: 工单索引文件存放路径,当索引类型为 `bleve` 时有效。 | |||
- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: 工单索引队列类型,当前支持 `channel` 或 `levelqueue`。 | |||
- `ISSUE_INDEXER_QUEUE_DIR`: **indexers/issues.queue**: 当 `ISSUE_INDEXER_QUEUE_TYPE` 为 `levelqueue` 时,保存索引队列的磁盘路径。 | |||
- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: 队列处理中批量提交数量。 | |||
- `REPO_INDEXER_ENABLED`: **false**: 是否启用代码搜索(启用后会占用比较大的磁盘空间)。 | |||
- `REPO_INDEXER_PATH`: **indexers/repos.bleve**: 用于代码搜索的索引文件路径。 | |||
- `UPDATE_BUFFER_LEN`: **20**: 代码索引请求的缓冲区长度。 | |||
- `MAX_FILE_SIZE`: **1048576**: 进行解析的源代码文件的最大长度,小于该值时才会索引。 | |||
## Security (`security`) | |||
- `INSTALL_LOCK`: 是否允许运行安装向导,(跟管理员账号有关,十分重要)。 |
@@ -1684,6 +1684,40 @@ func GetRepoIssueStats(repoID, uid int64, filterMode int, isPull bool) (numOpen | |||
return openResult, closedResult | |||
} | |||
// SearchIssueIDsByKeyword search issues on database | |||
func SearchIssueIDsByKeyword(kw string, repoID int64, limit, start int) (int64, []int64, error) { | |||
var repoCond = builder.Eq{"repo_id": repoID} | |||
var subQuery = builder.Select("id").From("issue").Where(repoCond) | |||
var cond = builder.And( | |||
repoCond, | |||
builder.Or( | |||
builder.Like{"name", kw}, | |||
builder.Like{"content", kw}, | |||
builder.In("id", builder.Select("issue_id"). | |||
From("comment"). | |||
Where(builder.And( | |||
builder.Eq{"type": CommentTypeComment}, | |||
builder.In("issue_id", subQuery), | |||
builder.Like{"content", kw}, | |||
)), | |||
), | |||
), | |||
) | |||
var ids = make([]int64, 0, limit) | |||
err := x.Distinct("id").Table("issue").Where(cond).Limit(limit, start).Find(&ids) | |||
if err != nil { | |||
return 0, nil, err | |||
} | |||
total, err := x.Distinct("id").Table("issue").Where(cond).Count() | |||
if err != nil { | |||
return 0, nil, err | |||
} | |||
return total, ids, nil | |||
} | |||
func updateIssue(e Engine, issue *Issue) error { | |||
_, err := e.ID(issue.ID).AllCols().Update(issue) | |||
if err != nil { |
@@ -295,3 +295,28 @@ func TestIssue_loadTotalTimes(t *testing.T) { | |||
assert.NoError(t, ms.loadTotalTimes(x)) | |||
assert.Equal(t, int64(3662), ms.TotalTrackedTime) | |||
} | |||
func TestIssue_SearchIssueIDsByKeyword(t *testing.T) { | |||
assert.NoError(t, PrepareTestDatabase()) | |||
total, ids, err := SearchIssueIDsByKeyword("issue2", 1, 10, 0) | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, 1, total) | |||
assert.EqualValues(t, []int64{2}, ids) | |||
total, ids, err = SearchIssueIDsByKeyword("first", 1, 10, 0) | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, 1, total) | |||
assert.EqualValues(t, []int64{1}, ids) | |||
total, ids, err = SearchIssueIDsByKeyword("for", 1, 10, 0) | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, 4, total) | |||
assert.EqualValues(t, []int64{1, 2, 3, 5}, ids) | |||
// issue1's comment id 2 | |||
total, ids, err = SearchIssueIDsByKeyword("good", 1, 10, 0) | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, 1, total) | |||
assert.EqualValues(t, []int64{1}, ids) | |||
} |
@@ -0,0 +1,45 @@ | |||
// Copyright 2019 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
package issues | |||
import "code.gitea.io/gitea/models" | |||
// DBIndexer implements Indexer inteface to use database's like search | |||
type DBIndexer struct { | |||
} | |||
// Init dummy function | |||
func (db *DBIndexer) Init() (bool, error) { | |||
return false, nil | |||
} | |||
// Index dummy function | |||
func (db *DBIndexer) Index(issue []*IndexerData) error { | |||
return nil | |||
} | |||
// Delete dummy function | |||
func (db *DBIndexer) Delete(ids ...int64) error { | |||
return nil | |||
} | |||
// Search dummy function | |||
func (db *DBIndexer) Search(kw string, repoID int64, limit, start int) (*SearchResult, error) { | |||
total, ids, err := models.SearchIssueIDsByKeyword(kw, repoID, limit, start) | |||
if err != nil { | |||
return nil, err | |||
} | |||
var result = SearchResult{ | |||
Total: total, | |||
Hits: make([]Match, 0, limit), | |||
} | |||
for _, id := range ids { | |||
result.Hits = append(result.Hits, Match{ | |||
ID: id, | |||
RepoID: repoID, | |||
}) | |||
} | |||
return &result, nil | |||
} |
@@ -33,7 +33,8 @@ type Match struct { | |||
// SearchResult represents search results | |||
type SearchResult struct { | |||
Hits []Match | |||
Total int64 | |||
Hits []Match | |||
} | |||
// Indexer defines an inteface to indexer issues contents | |||
@@ -54,6 +55,7 @@ var ( | |||
// all issue index done. | |||
func InitIssueIndexer(syncReindex bool) error { | |||
var populate bool | |||
var dummyQueue bool | |||
switch setting.Indexer.IssueType { | |||
case "bleve": | |||
issueIndexer = NewBleveIndexer(setting.Indexer.IssuePath) | |||
@@ -62,10 +64,17 @@ func InitIssueIndexer(syncReindex bool) error { | |||
return err | |||
} | |||
populate = !exist | |||
case "db": | |||
issueIndexer = &DBIndexer{} | |||
dummyQueue = true | |||
default: | |||
return fmt.Errorf("unknow issue indexer type: %s", setting.Indexer.IssueType) | |||
} | |||
if dummyQueue { | |||
return nil | |||
} | |||
var err error | |||
switch setting.Indexer.IssueIndexerQueueType { | |||
case setting.LevelQueueType: |
@@ -48,4 +48,8 @@ func TestSearchIssues(t *testing.T) { | |||
ids, err = SearchIssuesByKeyword(1, "for") | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, []int64{1, 2, 3, 5}, ids) | |||
ids, err = SearchIssuesByKeyword(1, "good") | |||
assert.NoError(t, err) | |||
assert.EqualValues(t, []int64{1}, ids) | |||
} |
@@ -7,5 +7,19 @@ package issues | |||
// Queue defines an interface to save an issue indexer queue | |||
type Queue interface { | |||
Run() error | |||
Push(*IndexerData) | |||
Push(*IndexerData) error | |||
} | |||
// DummyQueue represents an empty queue | |||
type DummyQueue struct { | |||
} | |||
// Run starts to run the queue | |||
func (b *DummyQueue) Run() error { | |||
return nil | |||
} | |||
// Push pushes data to indexer | |||
func (b *DummyQueue) Push(*IndexerData) error { | |||
return nil | |||
} |
@@ -33,6 +33,11 @@ func (c *ChannelQueue) Run() error { | |||
for { | |||
select { | |||
case data := <-c.queue: | |||
if data.IsDelete { | |||
c.indexer.Delete(data.IDs...) | |||
continue | |||
} | |||
datas = append(datas, data) | |||
if len(datas) >= c.batchNumber { | |||
c.indexer.Index(datas) | |||
@@ -51,6 +56,7 @@ func (c *ChannelQueue) Run() error { | |||
} | |||
// Push will push the indexer data to queue | |||
func (c *ChannelQueue) Push(data *IndexerData) { | |||
func (c *ChannelQueue) Push(data *IndexerData) error { | |||
c.queue <- data | |||
return nil | |||
} |
@@ -94,14 +94,10 @@ func (l *LevelQueue) Run() error { | |||
} | |||
// Push will push the indexer data to queue | |||
func (l *LevelQueue) Push(data *IndexerData) { | |||
func (l *LevelQueue) Push(data *IndexerData) error { | |||
bs, err := json.Marshal(data) | |||
if err != nil { | |||
log.Error(4, "Marshal: %v", err) | |||
return | |||
} | |||
err = l.queue.LPush(bs) | |||
if err != nil { | |||
log.Error(4, "LPush: %v", err) | |||
return err | |||
} | |||
return l.queue.LPush(bs) | |||
} |
@@ -38,6 +38,7 @@ var ( | |||
func newIndexerService() { | |||
sec := Cfg.Section("indexer") | |||
Indexer.IssueType = sec.Key("ISSUE_INDEXER_TYPE").MustString("bleve") | |||
Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString(path.Join(AppDataPath, "indexers/issues.bleve")) | |||
if !filepath.IsAbs(Indexer.IssuePath) { | |||
Indexer.IssuePath = path.Join(AppWorkPath, Indexer.IssuePath) |