Merge pull request #2711 from HeinleinSupport/master

Oletools,ICAP support / lua_scanners enhancements

1 files changed, 58 insertions, 27 deletions

diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua
index b6c3d552f..e5c0a1964 100644
--- a/lualib/lua_scanners/dcc.lua
+++ b/lualib/lua_scanners/dcc.lua
@@ -29,7 +29,7 @@ local fun = require "fun"
 
 local N = 'dcc'
 
-local function dcc_check(task, content, _, rule)
+local function dcc_check(task, content, digest, rule)
   local function dcc_check_uncached ()
     local upstream = rule.upstreams:get_upstream_round_robin()
     local addr = upstream:get_addr()
@@ -81,8 +81,7 @@ local function dcc_check(task, content, _, rule)
 
     local function dcc_callback(err, data, conn)
 
-      if err then
-
+      local function dcc_requery()
         -- set current upstream to fail because an error occurred
         upstream:fail()
 
@@ -91,11 +90,15 @@ local function dcc_check(task, content, _, rule)
 
           retransmits = retransmits - 1
 
+          lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s',
+            rule.log_prefix, err, retransmits)
+
           -- Select a different upstream!
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s: retry IP: %s', rule.log_prefix, addr)
+          lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s',
+            rule.log_prefix, addr, addr:get_port())
 
           tcp.request({
             task = task,
@@ -110,35 +113,35 @@ local function dcc_check(task, content, _, rule)
             fuz2_max = 999999,
           })
         else
-          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+            'exceed', rule.log_prefix)
+          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '..
+            'retransmits exceed')
         end
+      end
+
+      if err then
+
+        dcc_requery()
+
       else
         -- Parse the response
         if upstream then upstream:ok() end
         local _,_,result,disposition,header = tostring(data):find("(.-)\n(.-)\n(.-)\n")
-        lua_util.debugm(N, task, 'DCC result=%1 disposition=%2 header="%3"',
+        lua_util.debugm(rule.N, task, 'DCC result=%1 disposition=%2 header="%3"',
             result, disposition, header)
 
         if header then
           local _,_,info = header:find("; (.-)$")
-
           if (result == 'R') then
             -- Reject
             common.yield_result(task, rule, info, rule.default_score)
+            common.save_av_cache(task, digest, rule, info, rule.default_score)
           elseif (result == 'T') then
             -- Temporary failure
             rspamd_logger.warnx(task, 'DCC returned a temporary failure result: %s', result)
-            task:insert_result(rule.symbol_fail,
-                0.0,
-                'tempfail:' .. result)
+            dcc_requery()
           elseif result == 'A' then
-            if rule.log_clean then
-              rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
-                  rule.log_prefix, info)
-            else
-              lua_util.debugm(N, task, '%s: returned result A - info: %s',
-                  rule.log_prefix, info)
 
               local opts = {}
               local score = 0.0
@@ -188,25 +191,36 @@ local function dcc_check(task, content, _, rule)
                 task:insert_result(rule.symbol_bulk,
                     score,
                     opts)
+                common.save_av_cache(task, digest, rule, opts, score)
+              else
+                common.save_av_cache(task, digest, rule, 'OK')
+                if rule.log_clean then
+                  rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
+                      rule.log_prefix, info)
+                else
+                  lua_util.debugm(rule.N, task, '%s: returned result A - info: %s',
+                      rule.log_prefix, info)
               end
             end
           elseif result == 'G' then
             -- do nothing
+            common.save_av_cache(task, digest, rule, 'OK')
             if rule.log_clean then
               rspamd_logger.infox(task, '%s: clean, returned result G - info: %s', rule.log_prefix, info)
             else
-              lua_util.debugm(N, task, '%s: returned result G - info: %s', rule.log_prefix, info)
+              lua_util.debugm(rule.N, task, '%s: returned result G - info: %s', rule.log_prefix, info)
             end
           elseif result == 'S' then
             -- do nothing
+            common.save_av_cache(task, digest, rule, 'OK')
             if rule.log_clean then
               rspamd_logger.infox(task, '%s: clean, returned result S - info: %s', rule.log_prefix, info)
             else
-              lua_util.debugm(N, task, '%s: returned result S - info: %s', rule.log_prefix, info)
+              lua_util.debugm(rule.N, task, '%s: returned result S - info: %s', rule.log_prefix, info)
             end
           else
             -- Unknown result
-            rspamd_logger.warnx(task, 'DCC result error: %1', result);
+            rspamd_logger.warnx(task, '%s: result error: %1', rule.log_prefix, result);
             task:insert_result(rule.symbol_fail,
                 0.0,
                 'error: ' .. result)
@@ -222,21 +236,30 @@ local function dcc_check(task, content, _, rule)
       timeout = rule.timeout or 2.0,
       shutdown = true,
       data = request_data,
-      callback = dcc_callback
+      callback = dcc_callback,
+      body_max = 999999,
+      fuz1_max = 999999,
+      fuz2_max = 999999,
     })
   end
-  if common.need_av_check(task, content, rule, N) then
-    dcc_check_uncached()
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, dcc_check_uncached) then
+      return
+    else
+      dcc_check_uncached()
+    end
   end
 end
 
 local function dcc_config(opts)
 
   local dcc_conf = {
+    N = N,
     default_port = 10045,
     timeout = 5.0,
     log_clean = false,
     retransmits = 2,
+    cache_expire = 7200, -- expire redis in 2h
     message = '${SCANNER}: bulk message found: "${VIRUS}"',
     detection_category = "hash",
     default_score = 1,
@@ -252,8 +275,16 @@ local function dcc_config(opts)
 
   dcc_conf = lua_util.override_defaults(dcc_conf, opts)
 
+  if not dcc_conf.prefix then
+    dcc_conf.prefix = 'rs_' .. dcc_conf.name .. '_'
+  end
+
   if not dcc_conf.log_prefix then
-    dcc_conf.log_prefix = N
+    if dcc_conf.name:lower() == dcc_conf.type:lower() then
+      dcc_conf.log_prefix = dcc_conf.name
+    else
+      dcc_conf.log_prefix = dcc_conf.name .. ' (' .. dcc_conf.type .. ')'
+    end
   end
 
   if not dcc_conf.servers and dcc_conf.socket then
@@ -271,7 +302,7 @@ local function dcc_config(opts)
       dcc_conf.default_port)
 
   if dcc_conf.upstreams then
-    lua_util.add_debug_alias('external_services', N)
+    lua_util.add_debug_alias('external_services', dcc_conf.N)
     return dcc_conf
   end
 
@@ -285,5 +316,5 @@ return {
   description = 'dcc bulk scanner',
   configure = dcc_config,
   check = dcc_check,
-  name = 'dcc'
-}
\ No newline at end of file
+  name = N
+}