[Fix] Fuzzy_check: Disable shingles for short texts (really)

diff --git a/conf/modules.d/fuzzy_check.conf b/conf/modules.d/fuzzy_check.conf
index 5f02d864bacb3473e0acfdda361c94f07af9873a..73e280f7958471e65d236f232cdd2154cce5902a 100644 (file)
--- a/conf/modules.d/fuzzy_check.conf
+++ b/conf/modules.d/fuzzy_check.conf
@@ -25,7 +25,8 @@ fuzzy_check {
     max_score = 20.0;
     read_only = yes;
     skip_unknown = yes;
-    short_text_direct_hash = true;
+    short_text_direct_hash = true; # If less than min_length then use direct hash
+    min_length = 64; # Minimum words count to consider shingles
     fuzzy_map = {
       FUZZY_DENIED {
         max_score = 20.0;

diff --git a/lualib/lua_fuzzy.lua b/lualib/lua_fuzzy.lua
index d2733d5d6012520f4ccf49354810374d00780133..0131ef8e275143da419c9bd6d21c18219163c9d8 100644 (file)
--- a/lualib/lua_fuzzy.lua
+++ b/lualib/lua_fuzzy.lua
@@ -157,14 +157,18 @@ local function check_text_part(task, part, rule, text)
 
   if rule.text_shingles then
     -- Check number of words
-    if rule.min_length > 0 and wcnt < rule.min_length then
+    local min_words = rule.min_length or 0
+    if min_words < 32 then
+      min_words = 32 -- Minimum for shingles
+    end
+    if wcnt < min_words then
       lua_util.debugm(N, task, 'text has less than %s words: %s; disable shingles',
           rule.min_length, wcnt)
       allow_shingles = false
     else
       lua_util.debugm(N, task, 'allow shingles in text %s, %s words',
           id, wcnt)
-      allow_shingles = wcnt > 0
+      allow_shingles = true
     end
 
     if not rule.short_text_direct_hash and not allow_shingles then
@@ -191,7 +195,7 @@ end
 local function has_sane_text_parts(task)
   local text_parts = task:get_text_parts() or {}
 
-  return fun.any(function(tp) return tp:get_words_count() > 10 end, text_parts)
+  return fun.any(function(tp) return tp:get_words_count() > 32 end, text_parts)
 end
 
 local function check_image_part(task, part, rule, image)