[Minor] lua_scanners - simplify need_check and dynamic_scan

11 files changed, 125 insertions, 160 deletions

diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua
index 01386cfe7..7771261e1 100644
--- a/lualib/lua_scanners/clamav.lua
+++ b/lualib/lua_scanners/clamav.lua
@@ -151,19 +151,11 @@ local function clamav_check(task, content, digest, rule)
           end
         end
         if cached then
-          common.save_av_cache(task, digest, rule, cached)
+          common.save_cache(task, digest, rule, cached)
         end
       end
     end
 
-    if rule.dynamic_scan then
-      local pre_check, pre_check_msg = common.check_metric_results(task, rule)
-      if pre_check then
-        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
-        return true
-      end
-    end
-
     tcp.request({
       task = task,
       host = addr:to_string(),
@@ -175,13 +167,10 @@ local function clamav_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule) then
-    if common.check_av_cache(task, digest, rule, clamav_check_uncached) then
-      return
-    else
-      clamav_check_uncached()
-    end
+  if common.need_check(task, content, rule, digest) then
+    clamav_check_uncached()
   end
+
 end
 
 return {
diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua
index 65dd4aef8..dcd31db30 100644
--- a/lualib/lua_scanners/common.lua
+++ b/lualib/lua_scanners/common.lua
@@ -125,11 +125,68 @@ local function message_not_too_large(task, content, rule)
   return true
 end
 
-local function need_av_check(task, content, rule)
-  return message_not_too_large(task, content, rule)
+local function message_not_too_small(task, content, rule)
+  local min_size = tonumber(rule.min_size)
+  if not min_size then return true end
+  if #content < min_size then
+    rspamd_logger.infox(task, "skip %s check as it is too small: %s (%s is allowed)",
+        rule.log_prefix, #content, min_size)
+    return false
+  end
+  return true
+end
+
+local function message_min_words(task, rule)
+  if rule.text_part_min_words then
+    local text_parts_empty = false
+    local text_parts = task:get_text_parts()
+
+    local filter_func = function(p)
+      return p:get_words_count() <= tonumber(rule.text_part_min_words)
+    end
+
+    fun.each(function(p)
+      text_parts_empty = true
+      rspamd_logger.infox(task, '%s: #words is less then text_part_min_words: %s',
+        rule.log_prefix, rule.text_part_min_words)
+    end, fun.filter(filter_func, text_parts))
+
+    return text_parts_empty
+  else
+    return true
+  end
 end
 
-local function check_av_cache(task, digest, rule, fn)
+local function dynamic_scan(task, rule)
+  if rule.dynamic_scan then
+    if rule.action ~= 'reject' then
+      local metric_result = task:get_metric_score('default')
+      local metric_action = task:get_metric_action('default')
+      local has_pre_result = task:has_pre_result()
+      -- ToDo: needed?
+      -- Sometimes leads to FPs
+      --if rule.symbol_type == 'postfilter' and metric_action == 'reject' then
+      --  rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, "result is already reject")
+      --  return false
+      --elseif metric_result[1] > metric_result[2]*2 then
+      if metric_result[1] > metric_result[2]*2 then
+        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'score > 2 * reject_level: ' .. metric_result[1])
+        return false
+      elseif has_pre_result and metric_action == 'reject' then
+        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'pre_result reject is set')
+        return false
+      else
+        return true, 'undecided'
+      end
+    else
+      return true, 'dynamic_scan is not possible with config `action=reject;`'
+    end
+  else
+    return true
+  end
+end
+
+local function check_cache(task, digest, rule, fn)
   local key = digest
 
   local function redis_av_cb(err, data)
@@ -173,7 +230,15 @@ local function check_av_cache(task, digest, rule, fn)
   return false
 end
 
-local function save_av_cache(task, digest, rule, to_save, dyn_weight)
+local function need_check(task, content, rule, digest)
+  return check_cache(task, digest, rule) and
+    message_not_too_large(task, content, rule) and
+    message_not_too_small(task, content, rule) and
+    message_min_words(task, rule) and
+    dynamic_scan(task, rule)
+end
+
+local function save_cache(task, digest, rule, to_save, dyn_weight)
   local key = digest
   if not dyn_weight then dyn_weight = 1.0 end
 
@@ -363,9 +428,9 @@ end
 exports.log_clean = log_clean
 exports.yield_result = yield_result
 exports.match_patterns = match_patterns
-exports.need_av_check = need_av_check
-exports.check_av_cache = check_av_cache
-exports.save_av_cache = save_av_cache
+exports.need_check = need_check
+exports.check_cache = check_cache
+exports.save_cache = save_cache
 exports.create_regex_table = create_regex_table
 exports.check_parts_match = check_parts_match
 exports.check_metric_results = check_metric_results
diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua
index e26e666f9..85d72dd8b 100644
--- a/lualib/lua_scanners/dcc.lua
+++ b/lualib/lua_scanners/dcc.lua
@@ -193,7 +193,7 @@ local function dcc_check(task, content, digest, rule)
           if (result == 'R') then
             -- Reject
             common.yield_result(task, rule, info, rule.default_score)
-            common.save_av_cache(task, digest, rule, info, rule.default_score)
+            common.save_cache(task, digest, rule, info, rule.default_score)
           elseif (result == 'T') then
             -- Temporary failure
             rspamd_logger.warnx(task, 'DCC returned a temporary failure result: %s', result)
@@ -248,9 +248,9 @@ local function dcc_check(task, content, digest, rule)
                 task:insert_result(rule.symbol_bulk,
                     score,
                     opts)
-                common.save_av_cache(task, digest, rule, opts, score)
+                common.save_cache(task, digest, rule, opts, score)
               else
-                common.save_av_cache(task, digest, rule, 'OK')
+                common.save_cache(task, digest, rule, 'OK')
                 if rule.log_clean then
                   rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
                       rule.log_prefix, info)
@@ -261,7 +261,7 @@ local function dcc_check(task, content, digest, rule)
             end
           elseif result == 'G' then
             -- do nothing
-            common.save_av_cache(task, digest, rule, 'OK')
+            common.save_cache(task, digest, rule, 'OK')
             if rule.log_clean then
               rspamd_logger.infox(task, '%s: clean, returned result G - info: %s', rule.log_prefix, info)
             else
@@ -269,7 +269,7 @@ local function dcc_check(task, content, digest, rule)
             end
           elseif result == 'S' then
             -- do nothing
-            common.save_av_cache(task, digest, rule, 'OK')
+            common.save_cache(task, digest, rule, 'OK')
             if rule.log_clean then
               rspamd_logger.infox(task, '%s: clean, returned result S - info: %s', rule.log_prefix, info)
             else
@@ -284,14 +284,6 @@ local function dcc_check(task, content, digest, rule)
       end
     end
 
-    if rule.dynamic_scan then
-      local pre_check, pre_check_msg = common.check_metric_results(task, rule)
-      if pre_check then
-        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
-        return true
-      end
-    end
-
     tcp.request({
       task = task,
       host = addr:to_string(),
@@ -305,13 +297,11 @@ local function dcc_check(task, content, digest, rule)
       fuz2_max = 999999,
     })
   end
-  if common.need_av_check(task, content, rule) then
-    if common.check_av_cache(task, digest, rule, dcc_check_uncached) then
-      return
-    else
-      dcc_check_uncached()
-    end
+
+  if common.need_check(task, content, rule, digest) then
+    dcc_check_uncached()
   end
+
 end
 
 return {
diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua
index 907fab139..30d179ec6 100644
--- a/lualib/lua_scanners/fprot.lua
+++ b/lualib/lua_scanners/fprot.lua
@@ -144,19 +144,11 @@ local function fprot_check(task, content, digest, rule)
           end
         end
         if cached then
-          common.save_av_cache(task, digest, rule, cached)
+          common.save_cache(task, digest, rule, cached)
         end
       end
     end
 
-    if rule.dynamic_scan then
-      local pre_check, pre_check_msg = common.check_metric_results(task, rule)
-      if pre_check then
-        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
-        return true
-      end
-    end
-
     tcp.request({
       task = task,
       host = addr:to_string(),
@@ -168,13 +160,10 @@ local function fprot_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule) then
-    if common.check_av_cache(task, digest, rule, fprot_check_uncached) then
-      return
-    else
-      fprot_check_uncached()
-    end
+  if common.need_check(task, content, rule, digest) then
+    fprot_check_uncached()
   end
+
 end
 
 return {
diff --git a/lualib/lua_scanners/icap.lua b/lualib/lua_scanners/icap.lua
index 908c2c1f6..c7b495c29 100644
--- a/lualib/lua_scanners/icap.lua
+++ b/lualib/lua_scanners/icap.lua
@@ -354,14 +354,6 @@ local function icap_check(task, content, digest, rule)
       end
     end
 
-    if rule.dynamic_scan then
-      local pre_check, pre_check_msg = common.check_metric_results(task, rule)
-      if pre_check then
-        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
-        return true
-      end
-    end
-
     tcp.request({
       task = task,
       host = addr:to_string(),
@@ -374,13 +366,10 @@ local function icap_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule) then
-    if common.check_av_cache(task, digest, rule, icap_check_uncached) then
-      return
-    else
-      icap_check_uncached()
-    end
+  if common.need_check(task, content, rule, digest) then
+    icap_check_uncached()
   end
+
 end
 
 return {
diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua
index 87411c3b9..cb652f5b6 100644
--- a/lualib/lua_scanners/kaspersky_av.lua
+++ b/lualib/lua_scanners/kaspersky_av.lua
@@ -162,19 +162,11 @@ local function kaspersky_check(task, content, digest, rule)
           end
         end
         if cached then
-          common.save_av_cache(task, digest, rule, cached)
+          common.save_cache(task, digest, rule, cached)
         end
       end
     end
 
-    if rule.dynamic_scan then
-      local pre_check, pre_check_msg = common.check_metric_results(task, rule)
-      if pre_check then
-        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
-        return true
-      end
-    end
-
     tcp.request({
       task = task,
       host = addr:to_string(),
@@ -186,13 +178,10 @@ local function kaspersky_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule) then
-    if common.check_av_cache(task, digest, rule, kaspersky_check_uncached) then
-      return
-    else
-      kaspersky_check_uncached()
-    end
+  if common.need_check(task, content, rule, digest) then
+    kaspersky_check_uncached()
   end
+
 end
 
 return {
diff --git a/lualib/lua_scanners/oletools.lua b/lualib/lua_scanners/oletools.lua
index 3daa76713..88ecfdece 100644
--- a/lualib/lua_scanners/oletools.lua
+++ b/lualib/lua_scanners/oletools.lua
@@ -42,6 +42,7 @@ local function oletools_config(opts)
     log_clean = false,
     retransmits = 2,
     cache_expire = 86400, -- expire redis in 1d
+    min_size = 500,
     symbol = "OLETOOLS",
     message = '${SCANNER}: Oletools threat message found: "${VIRUS}"',
     detection_category = "office macro",
@@ -176,7 +177,7 @@ local function oletools_check(task, content, digest, rule)
             rspamd_logger.errx(task, '%s: ERROR found: %s', rule.log_prefix,
                 result[1].error)
             if result[1].error == 'File too small' then
-              common.save_av_cache(task, digest, rule, 'OK')
+              common.save_cache(task, digest, rule, 'OK')
               common.log_clean(task, rule, 'File too small to be scanned for macros')
             else
               oletools_requery(result[1].error)
@@ -198,7 +199,7 @@ local function oletools_check(task, content, digest, rule)
             rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix)
             common.yield_result(task, rule, 'oletools unhandled error', 0.0, 'fail')
           elseif type(result[2]['analysis']) ~= 'table' and #result[2]['macros'] == 0 then
-            common.save_av_cache(task, digest, rule, 'OK')
+            common.save_cache(task, digest, rule, 'OK')
             common.log_clean(task, rule, 'No macro found')
           elseif #result[2]['macros'] > 0 then
             -- M=Macros, A=Auto-executable, S=Suspicious keywords, I=IOCs,
@@ -257,7 +258,7 @@ local function oletools_check(task, content, digest, rule)
               local threat = 'AutoExec + Suspicious (' .. table.concat(analysis_keyword_table, ',') .. ')'
               lua_util.debugm(rule.name, task, '%s: threat result: %s', rule.log_prefix, threat)
               common.yield_result(task, rule, threat, rule.default_score)
-              common.save_av_cache(task, digest, rule, threat, rule.default_score)
+              common.save_cache(task, digest, rule, threat, rule.default_score)
 
             elseif rule.extended == true and #analysis_keyword_table > 0 then
               -- report any flags (types) and any most keywords as individual virus name
@@ -276,9 +277,9 @@ local function oletools_check(task, content, digest, rule)
                   rule.log_prefix, table.concat(analysis_keyword_table, ','))
 
               common.yield_result(task, rule, analysis_keyword_table, rule.default_score)
-              common.save_av_cache(task, digest, rule, analysis_keyword_table, rule.default_score)
+              common.save_cache(task, digest, rule, analysis_keyword_table, rule.default_score)
             else
-              common.save_av_cache(task, digest, rule, 'OK')
+              common.save_cache(task, digest, rule, 'OK')
               common.log_clean(task, rule, 'Scanned Macro is OK')
             end
 
@@ -290,14 +291,6 @@ local function oletools_check(task, content, digest, rule)
       end
     end
 
-    if rule.dynamic_scan then
-      local pre_check, pre_check_msg = common.check_metric_results(task, rule)
-      if pre_check then
-        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
-        return true
-      end
-    end
-
     tcp.request({
       task = task,
       host = addr:to_string(),
@@ -309,13 +302,11 @@ local function oletools_check(task, content, digest, rule)
     })
 
   end
-  if common.need_av_check(task, content, rule) then
-    if common.check_av_cache(task, digest, rule, oletools_check_uncached) then
-      return
-    else
-      oletools_check_uncached()
-    end
+
+  if common.need_check(task, content, rule, digest) then
+    oletools_check_uncached()
   end
+
 end
 
 return {
diff --git a/lualib/lua_scanners/savapi.lua b/lualib/lua_scanners/savapi.lua
index 65a9c825c..11f658da3 100644
--- a/lualib/lua_scanners/savapi.lua
+++ b/lualib/lua_scanners/savapi.lua
@@ -127,7 +127,7 @@ local function savapi_check(task, content, digest, rule)
         end
 
         common.yield_result(task, rule, vname)
-        common.save_av_cache(task, digest, rule, vname)
+        common.save_cache(task, digest, rule, vname)
       end
       if conn then
         conn:close()
@@ -144,7 +144,7 @@ local function savapi_check(task, content, digest, rule)
         if rule['log_clean'] then
           rspamd_logger.infox(task, '%s: message or mime_part is clean', rule['type'])
         end
-        common.save_av_cache(task, digest, rule, 'OK')
+        common.save_cache(task, digest, rule, 'OK')
         conn:add_write(savapi_fin_cb, 'QUIT\n')
 
         -- Terminal response - infected
@@ -237,14 +237,6 @@ local function savapi_check(task, content, digest, rule)
       end
     end
 
-    if rule.dynamic_scan then
-      local pre_check, pre_check_msg = common.check_metric_results(task, rule)
-      if pre_check then
-        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
-        return true
-      end
-    end
-
     tcp.request({
       task = task,
       host = addr:to_string(),
@@ -255,13 +247,10 @@ local function savapi_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule) then
-    if common.check_av_cache(task, digest, rule, savapi_check_uncached) then
-      return
-    else
-      savapi_check_uncached()
-    end
+  if common.need_check(task, content, rule, digest) then
+    savapi_check_uncached()
   end
+
 end
 
 return {
diff --git a/lualib/lua_scanners/sophos.lua b/lualib/lua_scanners/sophos.lua
index 59facc845..ff47c262d 100644
--- a/lualib/lua_scanners/sophos.lua
+++ b/lualib/lua_scanners/sophos.lua
@@ -125,7 +125,7 @@ local function sophos_check(task, content, digest, rule)
         local vname = string.match(data, 'VIRUS (%S+) ')
         if vname then
           common.yield_result(task, rule, vname)
-          common.save_av_cache(task, digest, rule, vname)
+          common.save_cache(task, digest, rule, vname)
         else
           if string.find(data, 'DONE OK') then
             if rule['log_clean'] then
@@ -134,7 +134,7 @@ local function sophos_check(task, content, digest, rule)
               lua_util.debugm(rule.name, task,
                   '%s: message or mime_part is clean', rule.log_prefix)
             end
-            common.save_av_cache(task, digest, rule, 'OK')
+            common.save_cache(task, digest, rule, 'OK')
             -- not finished - continue
           elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then
             conn:add_read(sophos_callback)
@@ -157,14 +157,6 @@ local function sophos_check(task, content, digest, rule)
       end
     end
 
-    if rule.dynamic_scan then
-      local pre_check, pre_check_msg = common.check_metric_results(task, rule)
-      if pre_check then
-        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
-        return true
-      end
-    end
-
     tcp.request({
       task = task,
       host = addr:to_string(),
@@ -175,13 +167,10 @@ local function sophos_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule) then
-    if common.check_av_cache(task, digest, rule, sophos_check_uncached) then
-      return
-    else
-      sophos_check_uncached()
-    end
+  if common.need_check(task, content, rule, digest) then
+    sophos_check_uncached()
   end
+
 end
 
 return {
diff --git a/lualib/lua_scanners/spamassassin.lua b/lualib/lua_scanners/spamassassin.lua
index 860df42dd..dd4c914bb 100644
--- a/lualib/lua_scanners/spamassassin.lua
+++ b/lualib/lua_scanners/spamassassin.lua
@@ -177,14 +177,14 @@ local function spamassassin_check(task, content, digest, rule)
 
           if rule.extended == false then
             common.yield_result(task, rule, symbols, spam_score)
-            common.save_av_cache(task, digest, rule, symbols, spam_score)
+            common.save_cache(task, digest, rule, symbols, spam_score)
           else
             local symbols_table = {}
             symbols_table = rspamd_str_split(symbols, ",")
             lua_util.debugm(rule.N, task, '%s: returned symbols as table: %s', rule.log_prefix, symbols_table)
 
             common.yield_result(task, rule, symbols_table, spam_score)
-            common.save_av_cache(task, digest, rule, symbols_table, spam_score)
+            common.save_cache(task, digest, rule, symbols_table, spam_score)
           end
         else
           common.log_clean(task, rule, 'no spam detected - spam score: ' .. spam_score .. ', symbols: ' .. symbols)
@@ -192,14 +192,6 @@ local function spamassassin_check(task, content, digest, rule)
       end
     end
 
-    if rule.dynamic_scan then
-      local pre_check, pre_check_msg = common.check_metric_results(task, rule)
-      if pre_check then
-        rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
-        return true
-      end
-    end
-
     tcp.request({
       task = task,
       host = addr:to_string(),
@@ -209,13 +201,11 @@ local function spamassassin_check(task, content, digest, rule)
       callback = spamassassin_callback,
     })
   end
-  if common.need_av_check(task, content, rule) then
-    if common.check_av_cache(task, digest, rule, spamassassin_check_uncached) then
-      return
-    else
-      spamassassin_check_uncached()
-    end
+
+  if common.need_check(task, content, rule, digest) then
+    spamassassin_check_uncached()
   end
+
 end
 
 return {
diff --git a/lualib/lua_scanners/vadesecure.lua b/lualib/lua_scanners/vadesecure.lua
index 196d55ba4..0e539ef46 100644
--- a/lualib/lua_scanners/vadesecure.lua
+++ b/lualib/lua_scanners/vadesecure.lua
@@ -305,16 +305,11 @@ local function vade_check(task, content, digest, rule)
     end
   end
 
-  if rule.dynamic_scan then
-    local pre_check, pre_check_msg = common.check_metric_results(task, rule)
-    if pre_check then
-      rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
-      return true
-    end
+  if common.need_check(task, content, rule, digest) then
+    request_data.callback = vade_callback
+    http.request(request_data)
   end
 
-  request_data.callback = vade_callback
-  http.request(request_data)
 end
 
 return {