Fix bleve fuzziness (#30799) (#30804)

Backport #30799 by wxiaoguang

Fix #30797
Fix #30317

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go
index bd844205a6e804db8c511fd8b749655bfc0fcdf4..8056b58ec2350f0da05f99ff032121dde12d564d 100644 (file)
--- a/modules/indexer/code/bleve/bleve.go
+++ b/modules/indexer/code/bleve/bleve.go
@@ -39,8 +39,6 @@ import (
 const (
        unicodeNormalizeName = "unicodeNormalize"
        maxBatchSize         = 16
-       // fuzzyDenominator determines the levenshtein distance per each character of a keyword
-       fuzzyDenominator = 4
 )
 
 func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
@@ -245,7 +243,7 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
        phraseQuery.Analyzer = repoIndexerAnalyzer
        keywordQuery = phraseQuery
        if opts.IsKeywordFuzzy {
-               phraseQuery.Fuzziness = len(opts.Keyword) / fuzzyDenominator
+               phraseQuery.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword)
        }
 
        if len(opts.RepoIDs) > 0 {

diff --git a/modules/indexer/internal/bleve/util.go b/modules/indexer/internal/bleve/util.go
index 43a7c3c5ec1bdcc88addc672a6d48f5f6fc9e784..a2265f86e6b35dcbe77f836cc83028d110f46845 100644 (file)
--- a/modules/indexer/internal/bleve/util.go
+++ b/modules/indexer/internal/bleve/util.go
@@ -47,3 +47,15 @@ func openIndexer(path string, latestVersion int) (bleve.Index, int, error) {
 
        return index, 0, nil
 }
+
+func GuessFuzzinessByKeyword(s string) int {
+       // according to https://github.com/blevesearch/bleve/issues/1563, the supported max fuzziness is 2
+       // magic number 4 was chosen to determine the levenshtein distance per each character of a keyword
+       // BUT, when using CJK (eg: `갃갃갃` `啊啊啊`), it mismatches a lot.
+       for _, r := range s {
+               if r >= 128 {
+                       return 0
+               }
+       }
+       return min(2, len(s)/4)
+}

diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go
index 1f54be721b37c1c576ec867bfed1599e59a55c2a..d7957b266a2b7e96ff210b3c1c01cbf0e8faf3dc 100644 (file)
--- a/modules/indexer/issues/bleve/bleve.go
+++ b/modules/indexer/issues/bleve/bleve.go
@@ -35,11 +35,7 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
        })
 }
 
-const (
-       maxBatchSize = 16
-       // fuzzyDenominator determines the levenshtein distance per each character of a keyword
-       fuzzyDenominator = 4
-)
+const maxBatchSize = 16
 
 // IndexerData an update to the issue indexer
 type IndexerData internal.IndexerData
@@ -162,7 +158,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
        if options.Keyword != "" {
                fuzziness := 0
                if options.IsFuzzyKeyword {
-                       fuzziness = len(options.Keyword) / fuzzyDenominator
+                       fuzziness = inner_bleve.GuessFuzzinessByKeyword(options.Keyword)
                }
 
                queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{

diff --git a/routers/web/repo/search.go b/routers/web/repo/search.go
index 23cf898630f672bb7ede0beaf75bc083c10be869..d7854b2499b4b55385a100a2be5216cf38236974 100644 (file)
--- a/routers/web/repo/search.go
+++ b/routers/web/repo/search.go
@@ -28,6 +28,7 @@ func Search(ctx *context.Context) {
        ctx.Data["Language"] = language
        ctx.Data["IsFuzzy"] = isFuzzy
        ctx.Data["PageIsViewCode"] = true
+       ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
 
        if keyword == "" {
                ctx.HTML(http.StatusOK, tplSearch)
@@ -86,7 +87,6 @@ func Search(ctx *context.Context) {
                }
        }
 
-       ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
        ctx.Data["Repo"] = ctx.Repo.Repository
        ctx.Data["SearchResults"] = searchResults
        ctx.Data["SearchResultLanguages"] = searchResultLanguages