Merge pull request #2718 from HeinleinSupport/symbol_fail

[Minor] lua_scanners - use pattern for FAIL symbol

12 files changed, 59 insertions, 46 deletions

diff --git a/conf/modules.d/antivirus.conf b/conf/modules.d/antivirus.conf
index 803820dbb..e48f9329a 100644
--- a/conf/modules.d/antivirus.conf
+++ b/conf/modules.d/antivirus.conf
@@ -45,6 +45,10 @@ antivirus {
       # symbol_name = "pattern";
       JUST_EICAR = '^Eicar-Test-Signature$';
     }
+    patterns_fail {
+      # symbol_name = "pattern";
+      #CLAM_PROTOCOL_ERROR = '^unhandled response';
+    }
     # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
     whitelist = "/etc/rspamd/antivirus.wl";
   }
diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua
index 0f97028ea..b3a1b20f2 100644
--- a/lualib/lua_scanners/clamav.lua
+++ b/lualib/lua_scanners/clamav.lua
@@ -117,7 +117,7 @@ local function clamav_check(task, content, digest, rule)
           })
         else
           rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', rule.log_prefix)
-          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
 
       else
@@ -136,12 +136,18 @@ local function clamav_check(task, content, digest, rule)
           end
         else
           local vname = string.match(data, 'stream: (.+) FOUND')
-          if vname then
+          if string.find(vname, '^Heuristics%.Encrypted') then
+            rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix)
+            common.yield_result(task, rule, 'File is encrypted: '.. vname, 0.0, 'fail')
+          elseif string.find(vname, '^Heuristics%.Limits%.Exceeded') then
+            rspamd_logger.errx(task, '%s: ClamAV Limits Exceeded', rule.log_prefix)
+            common.yield_result(task, rule, 'Limits Exceeded: '.. vname, 0.0, 'fail')
+          elseif vname then
             common.yield_result(task, rule, vname)
             cached = vname
           else
             rspamd_logger.errx(task, '%s: unhandled response: %s', rule.log_prefix, data)
-            task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response')
+            common.yield_result(task, rule, 'unhandled response:' .. vname, 0.0, 'fail')
           end
         end
         if cached then
diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua
index cda656849..de5f77db9 100644
--- a/lualib/lua_scanners/common.lua
+++ b/lualib/lua_scanners/common.lua
@@ -61,13 +61,27 @@ local function match_patterns(default_sym, found, patterns, dyn_weight)
   end
 end
 
-local function yield_result(task, rule, vname, dyn_weight)
+local function yield_result(task, rule, vname, dyn_weight, is_fail)
   local all_whitelisted = true
-  if not dyn_weight then dyn_weight = 1.0 end
+  local patterns
+  local symbol
+
+  if not is_fail then
+    patterns = rule.patterns
+    symbol = rule.symbol
+    if not dyn_weight then dyn_weight = 1.0 end
+    lua_util.debugm(rule.name, task, '%s: no fail: %s',rule.log_prefix, symbol)
+  elseif is_fail == 'fail' then
+    patterns = rule.patterns_fail
+    symbol = rule.symbol_fail
+    dyn_weight = 0.0
+    lua_util.debugm(rule.name, task, '%s: FAIL: %s',rule.log_prefix, symbol)
+  end
+
   if type(vname) == 'string' then
-    local symname, symscore = match_patterns(rule.symbol,
+    local symname, symscore = match_patterns(symbol,
         vname,
-        rule.patterns,
+        patterns,
         dyn_weight)
     if rule.whitelist and rule.whitelist:get_key(vname) then
       rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vname)
@@ -78,7 +92,7 @@ local function yield_result(task, rule, vname, dyn_weight)
         rule.log_prefix, rule.detection_category, vname, symscore)
   elseif type(vname) == 'table' then
     for _, vn in ipairs(vname) do
-      local symname, symscore = match_patterns(rule.symbol, vn, rule.patterns, dyn_weight)
+      local symname, symscore = match_patterns(symbol, vn, patterns, dyn_weight)
       if rule.whitelist and rule.whitelist:get_key(vn) then
         rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vn)
       else
diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua
index 4ed149779..e775d698b 100644
--- a/lualib/lua_scanners/dcc.lua
+++ b/lualib/lua_scanners/dcc.lua
@@ -115,8 +115,7 @@ local function dcc_check(task, content, digest, rule)
         else
           rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
             'exceed', rule.log_prefix)
-          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '..
-            'retransmits exceed')
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
       end
 
@@ -221,9 +220,7 @@ local function dcc_check(task, content, digest, rule)
           else
             -- Unknown result
             rspamd_logger.warnx(task, '%s: result error: %1', rule.log_prefix, result);
-            task:insert_result(rule.symbol_fail,
-                0.0,
-                'error: ' .. result)
+            common.yield_result(task, rule, 'error: ' .. result, 0.0, 'fail')
           end
         end
       end
diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua
index f002f72c7..e624bc6fd 100644
--- a/lualib/lua_scanners/fprot.lua
+++ b/lualib/lua_scanners/fprot.lua
@@ -118,8 +118,7 @@ local function fprot_check(task, content, digest, rule)
           rspamd_logger.errx(task,
               '%s [%s]: failed to scan, maximum retransmits exceed',
               rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0,
-              'failed to scan and retransmits exceed')
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
       else
         upstream:ok()
diff --git a/lualib/lua_scanners/icap.lua b/lualib/lua_scanners/icap.lua
index 1e913211c..6a16dea73 100644
--- a/lualib/lua_scanners/icap.lua
+++ b/lualib/lua_scanners/icap.lua
@@ -137,12 +137,12 @@ local function icap_check(task, content, digest, rule)
           ICAP/1.0 500 Server error
         ]]--
         rspamd_logger.errx(task, '%s: ICAP ERROR: %s', rule.log_prefix, icap_headers.icap)
-        task:insert_result(rule.symbol_fail, 0.0, icap_headers.icap)
+        common.yield_result(task, rule, icap_headers.icap, 0.0, 'fail')
         return false
       else
         rspamd_logger.errx(task, '%s: unhandled response |%s|',
           rule.log_prefix, string.gsub(result, "\r\n", ", "))
-        task:insert_result(rule.symbol_fail, 0.0, 'unhandled icap response: ' .. icap_headers.icap)
+        common.yield_result(task, rule, 'unhandled icap response: ' .. icap_headers.icap, 0.0, 'fail')
       end
     end
 
@@ -162,12 +162,12 @@ local function icap_check(task, content, digest, rule)
         else
           rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s',
             rule.log_prefix, icap_headers['Methods'])
-          task:insert_result(rule.symbol_fail, 0.0, 'NO RESPMOD')
+          common.yield_result(task, rule, 'NO RESPMOD', 0.0, 'fail')
         end
       else
         rspamd_logger.errx(task, '%s: OPTIONS query failed: %s',
           rule.log_prefix, icap_headers.icap)
-        task:insert_result(rule.symbol_fail, 0.0, 'OPTIONS query failed')
+        common.yield_result(task, rule, 'OPTIONS query failed', 0.0, 'fail')
       end
     end
 
@@ -206,7 +206,7 @@ local function icap_check(task, content, digest, rule)
         else
           rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
             'exceed - err: %s', rule.log_prefix, error)
-          task:insert_result(rule.symbol_fail, 0.0, 'failed - err: ' .. error)
+          common.yield_result(task, rule, 'failed - err: ' .. error, 0.0, 'fail')
         end
       end
 
diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua
index ebed710de..d87f78886 100644
--- a/lualib/lua_scanners/kaspersky_av.lua
+++ b/lualib/lua_scanners/kaspersky_av.lua
@@ -138,8 +138,7 @@ local function kaspersky_check(task, content, digest, rule)
           rspamd_logger.errx(task,
               '%s [%s]: failed to scan, maximum retransmits exceed',
               rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0,
-              'failed to scan and retransmits exceed')
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
 
       else
@@ -159,7 +158,7 @@ local function kaspersky_check(task, content, digest, rule)
             cached = vname
           else
             rspamd_logger.errx(task, 'unhandled response: %s', data)
-            task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response')
+            common.yield_result(task, rule, 'unhandled response', 0.0, 'fail')
           end
         end
         if cached then
diff --git a/lualib/lua_scanners/oletools.lua b/lualib/lua_scanners/oletools.lua
index bd6cc9007..7ecea5dbc 100644
--- a/lualib/lua_scanners/oletools.lua
+++ b/lualib/lua_scanners/oletools.lua
@@ -71,7 +71,7 @@ local function oletools_check(task, content, digest, rule)
         else
           rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
               'exceed - err: %s', rule.log_prefix, error)
-          task:insert_result(rule.symbol_fail, 0.0, 'failed - err: ' .. error)
+          common.yield_result(task, rule, 'failed to scan, maximum retransmits exceed - err: ' .. error, 0.0, 'fail')
         end
       end
 
@@ -119,19 +119,20 @@ local function oletools_check(task, content, digest, rule)
           end
         elseif result[3]['return_code'] == 9 then
           rspamd_logger.warnx(task, '%s: File is encrypted.', rule.log_prefix)
+          common.yield_result(task, rule, 'failed - err: ' .. oletools_rc[result[3]['return_code']], 0.0, 'fail')
         elseif result[3]['return_code'] > 6 then
           rspamd_logger.errx(task, '%s: Error Returned: %s',
               rule.log_prefix, oletools_rc[result[3]['return_code']])
           rspamd_logger.errx(task, '%s: Error message: %s',
               rule.log_prefix, result[2]['message'])
-          task:insert_result(rule.symbol_fail, 0.0, 'failed - err: ' .. oletools_rc[result[3]['return_code']])
+          common.yield_result(task, rule, 'failed - err: ' .. oletools_rc[result[3]['return_code']], 0.0, 'fail')
         elseif result[3]['return_code'] > 1 then
           rspamd_logger.errx(task, '%s: Error message: %s',
               rule.log_prefix, result[2]['message'])
           oletools_requery(oletools_rc[result[3]['return_code']])
         elseif #result[2]['analysis'] == 0 and #result[2]['macros'] == 0 then
           rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix)
-          task:insert_result(rule.symbol_fail, 0.0, 'oletools unhandled error')
+          common.yield_result(task, rule, 'oletools unhandled error', 0.0, 'fail')
         elseif result[2]['analysis'] == 'null' and #result[2]['macros'] == 0 then
           common.save_av_cache(task, digest, rule, 'OK')
           common.log_clean(task, rule, 'No macro found')
@@ -218,7 +219,7 @@ local function oletools_check(task, content, digest, rule)
 
         else
           rspamd_logger.warnx(task, '%s: unhandled response', rule.log_prefix)
-          task:insert_result(rule.symbol_fail, 0.0, 'unhandled response')
+          common.yield_result(task, rule, 'unhandled error', 0.0, 'fail')
         end
       end
     end
diff --git a/lualib/lua_scanners/savapi.lua b/lualib/lua_scanners/savapi.lua
index 4a7b7082a..13dbb7136 100644
--- a/lualib/lua_scanners/savapi.lua
+++ b/lualib/lua_scanners/savapi.lua
@@ -160,6 +160,7 @@ local function savapi_check(task, content, digest, rule)
           if not virus then
             rspamd_logger.errx(task, "%s: virus result unparseable: %s",
                 rule['type'], result)
+            common.yield_result(task, rule, 'virus result unparseable: ' .. result, 0.0, 'fail')
             return
           end
         end
@@ -185,6 +186,7 @@ local function savapi_check(task, content, digest, rule)
       else
         rspamd_logger.errx(task, '%s: invalid product id %s', rule['type'],
             rule['product_id'])
+        common.yield_result(task, rule, 'invalid product id: ' .. result, 0.0, 'fail')
         conn:add_write(savapi_fin_cb, 'QUIT\n')
       end
     end
@@ -222,7 +224,7 @@ local function savapi_check(task, content, digest, rule)
           })
         else
           rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
       else
         upstream:ok()
diff --git a/lualib/lua_scanners/sophos.lua b/lualib/lua_scanners/sophos.lua
index 934ce1f79..159e8abdc 100644
--- a/lualib/lua_scanners/sophos.lua
+++ b/lualib/lua_scanners/sophos.lua
@@ -117,7 +117,7 @@ local function sophos_check(task, content, digest, rule)
           })
         else
           rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+          common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
         end
       else
         upstream:ok()
@@ -140,30 +140,19 @@ local function sophos_check(task, content, digest, rule)
             -- not finished - continue
           elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then
             conn:add_read(sophos_callback)
-            -- set pseudo virus if configured, else do nothing since it's no fatal
           elseif string.find(data, 'FAIL 0212') then
-            rspamd_logger.infox(task, 'Message is ENCRYPTED (0212 SOPHOS_SAVI_ERROR_FILE_ENCRYPTED): %s', data)
-            if rule['savdi_report_encrypted'] then
-              common.yield_result(task, rule, "SAVDI_FILE_ENCRYPTED")
-              common.save_av_cache(task, digest, rule, "SAVDI_FILE_ENCRYPTED")
-            end
-            -- set pseudo virus if configured, else set fail since part was not scanned
+            rspamd_logger.warnx(task, 'Message is encrypted (FAIL 0212): %s', data)
+            common.yield_result(task, rule, 'SAVDI: Message is encrypted (FAIL 0212)', 0.0, 'fail')
           elseif string.find(data, 'REJ 4') then
-            if rule['savdi_report_oversize'] then
-              rspamd_logger.infox(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data)
-              common.yield_result(task, rule, "SAVDI_FILE_OVERSIZED")
-              common.save_av_cache(task, digest, rule, "SAVDI_FILE_OVERSIZED")
-            else
-              rspamd_logger.errx(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data)
-              task:insert_result(rule['symbol_fail'], 0.0, 'Message is OVERSIZED (SSSP reject code 4):' .. data)
-            end
+            rspamd_logger.warnx(task, 'Message is oversized (REJ 4): %s', data)
+            common.yield_result(task, rule, 'SAVDI: Message oversized (REJ 4)', 0.0, 'fail')
             -- excplicitly set REJ1 message when SAVDIreports a protocol error
           elseif string.find(data, 'REJ 1') then
             rspamd_logger.errx(task, 'SAVDI (Protocol error (REJ 1)): %s', data)
-            task:insert_result(rule['symbol_fail'], 0.0, 'SAVDI (Protocol error (REJ 1)):' .. data)
+            common.yield_result(task, rule, 'SAVDI: Protocol error (REJ 1)', 0.0, 'fail')
           else
             rspamd_logger.errx(task, 'unhandled response: %s', data)
-            task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response')
+            common.yield_result(task, rule, 'unhandled response: ' .. data, 0.0, 'fail')
           end
 
         end
diff --git a/src/plugins/lua/antivirus.lua b/src/plugins/lua/antivirus.lua
index 0dde3e217..68dcedb64 100644
--- a/src/plugins/lua/antivirus.lua
+++ b/src/plugins/lua/antivirus.lua
@@ -108,6 +108,7 @@ local function add_antivirus_rule(sym, opts)
   end
 
   rule.patterns = common.create_regex_table(opts.patterns or {})
+  rule.patterns_fail = common.create_regex_table(opts.patterns_fail or {})
 
   if opts.whitelist then
     rule.whitelist = rspamd_config:add_hash_map(opts.whitelist)
diff --git a/src/plugins/lua/external_services.lua b/src/plugins/lua/external_services.lua
index c5889112f..de6615c86 100644
--- a/src/plugins/lua/external_services.lua
+++ b/src/plugins/lua/external_services.lua
@@ -148,6 +148,7 @@ local function add_scanner_rule(sym, opts)
   end
 
   rule.patterns = common.create_regex_table(opts.patterns or {})
+  rule.patterns_fail = common.create_regex_table(opts.patterns_fail or {})
 
   rule.mime_parts_filter_regex = common.create_regex_table(opts.mime_parts_filter_regex or {})