This PR improves the accuracy of Gitea's code search.
Currently, Gitea does not consider statements such as
`onsole.log("hello")` as hits when the user searches for `log`. The
culprit is how both ES and Bleve are tokenizing the file contents (in
both cases, `console.log` is a whole token).
In ES' case, we changed the tokenizer to
[simple_pattern_split](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-simplepatternsplit-tokenizer.html#:~:text=The%20simple_pattern_split%20tokenizer%20uses%20a,the%20tokenization%20is%20generally%20faster.).
In such a case, tokens are words formed by digits and letters. In
Bleve's case, it employs a
[letter](https://blevesearch.com/docs/Tokenizers/) tokenizer.
Resolves #32220
---------
Signed-off-by: Bruno Sofiato <bruno.sofiato@gmail.com>
"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
+ "github.com/blevesearch/bleve/v2/analysis/tokenizer/letter"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/mapping"
"github.com/blevesearch/bleve/v2/search/query"
filenameIndexerAnalyzer = "filenameIndexerAnalyzer"
filenameIndexerTokenizer = "filenameIndexerTokenizer"
repoIndexerDocType = "repoIndexerDocType"
- repoIndexerLatestVersion = 7
+ repoIndexerLatestVersion = 8
)
// generateBleveIndexMapping generates a bleve index mapping for the repo indexer
} else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{
"type": analyzer_custom.Name,
"char_filters": []string{},
- "tokenizer": unicode.Name,
+ "tokenizer": letter.Name,
"token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
}); err != nil {
return nil, err
)
const (
- esRepoIndexerLatestVersion = 2
+ esRepoIndexerLatestVersion = 3
// multi-match-types, currently only 2 types are used
// Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types
esMultiMatchTypeBestFields = "best_fields"
"settings": {
"analysis": {
"analyzer": {
+ "content_analyzer": {
+ "tokenizer": "content_tokenizer",
+ "filter" : ["lowercase"]
+ },
"filename_path_analyzer": {
"tokenizer": "path_tokenizer"
},
}
},
"tokenizer": {
+ "content_tokenizer": {
+ "type": "simple_pattern_split",
+ "pattern": "[^a-zA-Z0-9]"
+ },
"path_tokenizer": {
"type": "path_hierarchy",
"delimiter": "/"
"content": {
"type": "text",
"term_vector": "with_positions_offsets",
- "index": true
+ "index": true,
+ "analyzer": "content_analyzer"
},
"commit_id": {
"type": "keyword",
},
},
},
+ // Search for matches on the contents of files regardless of case.
+ {
+ RepoIDs: nil,
+ Keyword: "dESCRIPTION",
+ Langs: 1,
+ Results: []codeSearchResult{
+ {
+ Filename: "README.md",
+ Content: "# repo1\n\nDescription for repo1",
+ },
+ },
+ },
+ // Search for an exact match on the filename within the repo '62' (case insenstive).
+ // This scenario yields a single result (the file avocado.md on the repo '62')
+ {
+ RepoIDs: []int64{62},
+ Keyword: "AVOCADO.MD",
+ Langs: 1,
+ Results: []codeSearchResult{
+ {
+ Filename: "avocado.md",
+ Content: "# repo1\n\npineaple pie of cucumber juice",
+ },
+ },
+ },
+ // Search for matches on the contents of files when the criteria is a expression.
+ {
+ RepoIDs: []int64{62},
+ Keyword: "console.log",
+ Langs: 1,
+ Results: []codeSearchResult{
+ {
+ Filename: "example-file.js",
+ Content: "console.log(\"Hello, World!\")",
+ },
+ },
+ },
+ // Search for matches on the contents of files when the criteria is part of a expression.
+ {
+ RepoIDs: []int64{62},
+ Keyword: "log",
+ Langs: 1,
+ Results: []codeSearchResult{
+ {
+ Filename: "example-file.js",
+ Content: "console.log(\"Hello, World!\")",
+ },
+ },
+ },
}
for _, kw := range keywords {
import (
"errors"
"os"
+ "unicode"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
"github.com/blevesearch/bleve/v2"
- "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
+ unicode_tokenizer "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/index/upsidedown"
"github.com/ethantkoenig/rupture"
)
// may be different on two string and they still be considered equivalent.
// Given a phrasse, its shortest word determines its fuzziness. If a phrase uses CJK (eg: `갃갃갃` `啊啊啊`), the fuzziness is zero.
func GuessFuzzinessByKeyword(s string) int {
- tokenizer := unicode.NewUnicodeTokenizer()
+ tokenizer := unicode_tokenizer.NewUnicodeTokenizer()
tokens := tokenizer.Tokenize([]byte(s))
if len(tokens) > 0 {
// according to https://github.com/blevesearch/bleve/issues/1563, the supported max fuzziness is 2
// magic number 4 was chosen to determine the levenshtein distance per each character of a keyword
// BUT, when using CJK (eg: `갃갃갃` `啊啊啊`), it mismatches a lot.
+ // Likewise, queries whose terms contains characters that are *not* letters should not use fuzziness
+
for _, r := range s {
- if r >= 128 {
+ if r >= 128 || !unicode.IsLetter(r) {
return 0
}
}
Input: "갃갃갃",
Fuzziness: 0,
},
+ {
+ Input: "repo1",
+ Fuzziness: 0,
+ },
+ {
+ Input: "avocado.md",
+ Fuzziness: 0,
+ },
}
for _, scenario := range scenarios {
├── avocado.md
├── cucumber.md
├── ham.md
-└── potato
- └── ham.md
+├── potato
+| └── ham.md
+└── example-file.js
\ No newline at end of file
65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/develop
65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/feature/1
78fb907e3a3309eae4fe8fef030874cebbf1cd5e refs/heads/home-md-img-check
-3731fe53b763859aaf83e703ee731f6b9447ff1e refs/heads/master
+9f894b61946fd2f7b8b9d8e370e4d62f915522f5 refs/heads/master
62fb502a7172d4453f0322a2cc85bddffa57f07a refs/heads/pr-to-update
4649299398e4d39a5c09eb4f534df6f1e1eb87cc refs/heads/sub-home-md-img-check
3fa2f829675543ecfc16b2891aebe8bf0608a8f4 refs/notes/commits
-P pack-393dc29256bc27cb2ec73898507df710be7a3cf5.pack
+P pack-a7bef76cf6e2b46bc816936ab69306fb10aea571.pack
65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/develop
65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/feature/1
78fb907e3a3309eae4fe8fef030874cebbf1cd5e refs/heads/home-md-img-check
-3731fe53b763859aaf83e703ee731f6b9447ff1e refs/heads/master
+9f894b61946fd2f7b8b9d8e370e4d62f915522f5 refs/heads/master
62fb502a7172d4453f0322a2cc85bddffa57f07a refs/heads/pr-to-update
4649299398e4d39a5c09eb4f534df6f1e1eb87cc refs/heads/sub-home-md-img-check
3fa2f829675543ecfc16b2891aebe8bf0608a8f4 refs/notes/commits
projects / gitea.git / commitdiff