Merge pull request #2711 from HeinleinSupport/master

Oletools,ICAP support / lua_scanners enhancements

conf/groups.conf

@@ -107,5 +107,15 @@ group "neural" {
     .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/neural_group.conf"
 }
 
+group "antivirus" {
+    .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/antivirus_group.conf"
+    .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/antivirus_group.conf"
+}
+
+group "external_services" {
+    .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/external_services_group.conf"
+    .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/external_services_group.conf"
+}
+
 .include(try=true; priority=1; duplicate=merge) "$LOCAL_CONFDIR/local.d/groups.conf"
-.include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/groups.conf"
+.include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/groups.conf"

conf/modules.d/external_services.conf

@@ -0,0 +1,91 @@
+# Please don't modify this file as your changes might be overwritten with
+# the next update.
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
+# parameters defined on the top level
+#
+# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
+# parameters defined on the top level
+#
+# For specific modules or configuration you can also modify
+# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
+# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+#
+# See https://rspamd.com/doc/tutorials/writing_rules.html for details
+
+external_services {
+  oletools {
+    # If set force this action if any virus is found (default unset: no action is forced)
+    # action = "reject";
+    # If set, then rejection message is set to this value (mention single quotes)
+    # If `max_size` is set, messages > n bytes in size are not scanned
+    # max_size = 20000000;
+    # log_clean = true;
+    # servers = "127.0.0.1:10050";
+    # cache_expire = 86400;
+    # scan_mime_parts = true;
+    # extended = false;
+    # if `patterns` is specified virus name will be matched against provided regexes and the related
+    # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+    patterns {
+      # symbol_name = "pattern";
+      JUST_EICAR = "^Eicar-Test-Signature$";
+    }
+    # mime-part regex matching in content-type or filename
+    mime_parts_filter_regex {
+      #GEN1 = "application\/octet-stream";
+      DOC2 = "application\/msword";
+      DOC3 = "application\/vnd\.ms-word.*";
+      XLS = "application\/vnd\.ms-excel.*";
+      PPT = "application\/vnd\.ms-powerpoint.*";
+      GEN2 = "application\/vnd\.openxmlformats-officedocument.*";
+    }
+    # Mime-Part filename extension matching (no regex)
+    mime_parts_filter_ext {
+      doc = "doc";
+      dot = "dot";
+      docx = "docx";
+      dotx = "dotx";
+      docm = "docm";
+      dotm = "dotm";
+      xls = "xls";
+      xlt = "xlt";
+      xla = "xla";
+      xlsx = "xlsx";
+      xltx = "xltx";
+      xlsm = "xlsm";
+      xltm = "xltm";
+      xlam = "xlam";
+      xlsb = "xlsb";
+      ppt = "ppt";
+      pot = "pot";
+      pps = "pps";
+      ppa = "ppa";
+      pptx = "pptx";
+      potx = "potx";
+      ppsx = "ppsx";
+      ppam = "ppam";
+      pptm = "pptm";
+      potm = "potm";
+      ppsm = "ppsm";
+    }
+    # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+    whitelist = "/etc/rspamd/antivirus.wl";
+  }
+  dcc {
+    # If set force this action if any virus is found (default unset: no action is forced)
+    # action = "reject";
+    # If set, then rejection message is set to this value (mention single quotes)
+    # If `max_size` is set, messages > n bytes in size are not scanned
+    max_size = 20000000;
+    #servers = "127.0.0.1:10045";
+    # if `patterns` is specified virus name will be matched against provided regexes and the related
+    # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+    patterns {
+      # symbol_name = "pattern";
+      JUST_EICAR = "^Eicar-Test-Signature$";
+    }
+    # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+    whitelist = "/etc/rspamd/antivirus.wl";
+  }
+}

lualib/lua_scanners/clamav.lua

@@ -32,9 +32,10 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"'
 
 local function clamav_config(opts)
   local clamav_conf = {
-    scan_mime_parts = true;
-    scan_text_mime = false;
-    scan_image_mime = false;
+    N = N,
+    scan_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
     default_port = 3310,
     log_clean = false,
     timeout = 5.0, -- FIXME: this will break task_timeout!
@@ -44,12 +45,18 @@ local function clamav_config(opts)
     message = default_message,
   }
 
-  for k,v in pairs(opts) do
-    clamav_conf[k] = v
-  end
+  clamav_conf = lua_util.override_defaults(clamav_conf, opts)
 
   if not clamav_conf.prefix then
-    clamav_conf.prefix = 'rs_cl'
+    clamav_conf.prefix = 'rs_' .. clamav_conf.name .. '_'
+  end
+
+  if not clamav_conf.log_prefix then
+    if clamav_conf.name:lower() == clamav_conf.type:lower() then
+      clamav_conf.log_prefix = clamav_conf.name
+    else
+      clamav_conf.log_prefix = clamav_conf.name .. ' (' .. clamav_conf.type .. ')'
+    end
   end
 
   if not clamav_conf['servers'] then
@@ -63,7 +70,7 @@ local function clamav_config(opts)
       clamav_conf.default_port)
 
   if clamav_conf['upstreams'] then
-    lua_util.add_debug_alias('antivirus', N)
+    lua_util.add_debug_alias('antivirus', clamav_conf.N)
     return clamav_conf
   end
 
@@ -96,7 +103,7 @@ local function clamav_check(task, content, digest, rule)
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+          lua_util.debugm(rule.N, task, '%s: retry IP: %s', rule.log_prefix, addr)
 
           tcp.request({
             task = task,
@@ -108,7 +115,7 @@ local function clamav_check(task, content, digest, rule)
             stop_pattern = '\0'
           })
         else
-          rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
+          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', rule.log_prefix)
           task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
         end
 
@@ -116,26 +123,26 @@ local function clamav_check(task, content, digest, rule)
         upstream:ok()
         data = tostring(data)
         local cached
-        lua_util.debugm(N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data)
+        lua_util.debugm(rule.N, task, '%s: got reply: %s', rule.log_prefix, data)
         if data == 'stream: OK' then
           cached = 'OK'
           if rule['log_clean'] then
-            rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+            rspamd_logger.infox(task, '%s: message or mime_part is clean', rule.log_prefix)
           else
-            lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+            lua_util.debugm(rule.N, task, '%s: message or mime_part is clean', rule.log_prefix)
           end
         else
           local vname = string.match(data, 'stream: (.+) FOUND')
           if vname then
-            common.yield_result(task, rule, vname, N)
+            common.yield_result(task, rule, vname)
             cached = vname
           else
-            rspamd_logger.errx(task, 'unhandled response: %s', data)
+            rspamd_logger.errx(task, '%s: unhandled response: %s', rule.log_prefix, data)
             task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response')
           end
         end
         if cached then
-          common.save_av_cache(task, digest, rule, cached, N)
+          common.save_av_cache(task, digest, rule, cached)
         end
       end
     end
@@ -151,8 +158,8 @@ local function clamav_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule, N) then
-    if common.check_av_cache(task, digest, rule, clamav_check_uncached, N) then
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, clamav_check_uncached) then
       return
     else
       clamav_check_uncached()
@@ -165,5 +172,5 @@ return {
   description = 'clamav antivirus',
   configure = clamav_config,
   check = clamav_check,
-  name = 'clamav'
-}
+  name = N
+}

lualib/lua_scanners/common.lua

@@ -1,5 +1,6 @@
 --[[
 Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,30 +21,43 @@ limitations under the License.
 --]]
 
 local rspamd_logger = require "rspamd_logger"
+local rspamd_regexp = require "rspamd_regexp"
 local lua_util = require "lua_util"
 local lua_redis = require "lua_redis"
 local fun = require "fun"
 
 local exports = {}
 
-local function match_patterns(default_sym, found, patterns)
-  if type(patterns) ~= 'table' then return default_sym end
+local function log_clean(task, rule, msg)
+
+  msg = msg or 'message or mime_part is clean'
+
+  if rule.log_clean then
+    rspamd_logger.infox(task, '%s: %s', rule.log_prefix, msg)
+  else
+    lua_util.debugm(rule.module_name, task, '%s: %s', rule.log_prefix, msg)
+  end
+
+end
+
+local function match_patterns(default_sym, found, patterns, dyn_weight)
+  if type(patterns) ~= 'table' then return default_sym, dyn_weight end
   if not patterns[1] then
     for sym, pat in pairs(patterns) do
       if pat:match(found) then
-        return sym
+        return sym, '1'
       end
     end
-    return default_sym
+    return default_sym, dyn_weight
   else
     for _, p in ipairs(patterns) do
       for sym, pat in pairs(p) do
         if pat:match(found) then
-          return sym
+          return sym, '1'
         end
       end
     end
-    return default_sym
+    return default_sym, dyn_weight
   end
 end
 
@@ -51,23 +65,23 @@ local function yield_result(task, rule, vname, N, dyn_weight)
   local all_whitelisted = true
   if not dyn_weight then dyn_weight = 1.0 end
   if type(vname) == 'string' then
-    local symname = match_patterns(rule.symbol, vname, rule.patterns)
+    local symname, symscore = match_patterns(rule.symbol, vname, rule.patterns, dyn_weight)
     if rule.whitelist and rule.whitelist:get_key(vname) then
-      rspamd_logger.infox(task, '%s: "%s" is in whitelist', N, vname)
+      rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vname)
       return
     end
-    task:insert_result(symname, 1.0, vname)
-    rspamd_logger.infox(task, '%s: %s found: "%s"', N, rule.detection_category, vname)
+    task:insert_result(symname, symscore, vname)
+    rspamd_logger.infox(task, '%s: %s found: "%s"', rule.log_prefix, rule.detection_category, vname)
   elseif type(vname) == 'table' then
     for _, vn in ipairs(vname) do
-      local symname = match_patterns(rule.symbol, vn, rule.patterns)
+      local symname, symscore = match_patterns(rule.symbol, vn, rule.patterns, dyn_weight)
       if rule.whitelist and rule.whitelist:get_key(vn) then
-        rspamd_logger.infox(task, '%s: "%s" is in whitelist', N, vn)
+        rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vn)
       else
         all_whitelisted = false
-        task:insert_result(symname, dyn_weight, vn)
+        task:insert_result(symname, symscore, vn)
         rspamd_logger.infox(task, '%s: %s found: "%s"',
-            N, rule.detection_category, vn)
+            rule.log_prefix, rule.detection_category, vn)
       end
     end
   end
@@ -76,43 +90,45 @@ local function yield_result(task, rule, vname, N, dyn_weight)
       if all_whitelisted then return end
       vname = table.concat(vname, '; ')
     end
-    task:set_pre_result(rule['action'],
+    task:set_pre_result(rule.action,
         lua_util.template(rule.message or 'Rejected', {
-          SCANNER = N,
+          SCANNER = rule.name,
           VIRUS = vname,
         }), N)
   end
 end
 
-local function message_not_too_large(task, content, rule, N)
+local function message_not_too_large(task, content, rule)
   local max_size = tonumber(rule.max_size)
   if not max_size then return true end
   if #content > max_size then
     rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)",
-        N, #content, max_size)
+        rule.log_prefix, #content, max_size)
     return false
   end
   return true
 end
 
-local function need_av_check(task, content, rule, N)
-  return message_not_too_large(task, content, rule, N)
+local function need_av_check(task, content, rule)
+  return message_not_too_large(task, content, rule)
 end
 
-local function check_av_cache(task, digest, rule, fn, N)
+local function check_av_cache(task, digest, rule, fn)
   local key = digest
 
   local function redis_av_cb(err, data)
     if data and type(data) == 'string' then
       -- Cached
-      if data ~= 'OK' then
-        lua_util.debugm(N, task, 'got cached result for %s: %s',
-            key, data)
-        data = lua_util.str_split(data, '\v')
-        yield_result(task, rule, data, N)
+      data = rspamd_str_split(data, '\t')
+      local threat_string = rspamd_str_split(data[1], '\v')
+      local score = data[2] or rule.default_score
+      if threat_string[1] ~= 'OK' then
+        lua_util.debugm(rule.module_name, task, '%s: got cached threat result for %s: %s',
+          rule.log_prefix, key, threat_string[1])
+        yield_result(task, rule, threat_string, score)
       else
-        lua_util.debugm(N, task, 'got cached result for %s: %s',
-            key, data)
+        lua_util.debugm(rule.module_name, task, '%s: got cached negative result for %s: %s',
+          rule.log_prefix, key, threat_string[1])
       end
     else
       if err then
@@ -124,7 +140,7 @@ local function check_av_cache(task, digest, rule, fn, N)
 
   if rule.redis_params then
 
-    key = rule['prefix'] .. key
+    key = rule.prefix .. key
 
     if lua_redis.redis_make_request(task,
         rule.redis_params, -- connect params
@@ -141,7 +157,7 @@ local function check_av_cache(task, digest, rule, fn, N)
   return false
 end
 
-local function save_av_cache(task, digest, rule, to_save, N)
+local function save_av_cache(task, digest, rule, to_save, dyn_weight)
   local key = digest
 
   local function redis_set_cb(err)
@@ -150,8 +166,7 @@ local function save_av_cache(task, digest, rule, to_save, N)
       rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s',
           rule.detection_category, to_save, key, err)
     else
-      lua_util.debugm(N, task, 'saved cached result for %s: %s',
-          key, to_save)
+      lua_util.debugm(rule.module_name, task, '%s: saved cached result for %s: %s', rule.log_prefix, key, to_save)
     end
   end
 
@@ -159,6 +174,8 @@ local function save_av_cache(task, digest, rule, to_save, N)
     to_save = table.concat(to_save, '\v')
   end
 
+  local value = table.concat({to_save, dyn_weight}, '\t')
+
   if rule.redis_params and rule.prefix then
     key = rule.prefix .. key
 
@@ -168,29 +185,127 @@ local function save_av_cache(task, digest, rule, to_save, N)
         true, -- is write
         redis_set_cb, --callback
         'SETEX', -- command
-        { key, rule.cache_expire or 0, to_save }
+        { key, rule.cache_expire or 0, value }
     )
   end
 
   return false
 end
 
-local function text_parts_min_words(task, min_words)
-  local filter_func = function(p)
-    return p:get_words_count() >= min_words
+local function create_regex_table(patterns)
+  local regex_table = {}
+  if patterns[1] then
+    for i, p in ipairs(patterns) do
+      if type(p) == 'table' then
+        local new_set = {}
+        for k, v in pairs(p) do
+          new_set[k] = rspamd_regexp.create_cached(v)
+        end
+        regex_table[i] = new_set
+      else
+        regex_table[i] = {}
+      end
+    end
+  else
+    for k, v in pairs(patterns) do
+      regex_table[k] = rspamd_regexp.create_cached(v)
+    end
+  end
+  return regex_table
+end
+
+local function match_filter(task, found, patterns)
+  if type(patterns) ~= 'table' then return false end
+  if not patterns[1] then
+    for _, pat in pairs(patterns) do
+      if pat:match(found) then
+        return true
+      end
+    end
+    return false
+  else
+    for _, p in ipairs(patterns) do
+      for _, pat in ipairs(p) do
+        if pat:match(found) then
+          return true
+        end
+      end
+    end
+    return false
   end
+end
 
-  return fun.any(filter_func, task:get_text_parts())
+-- borrowed from mime_types.lua
+-- ext is the last extension, LOWERCASED
+-- ext2 is the one before last extension LOWERCASED
+local function gen_extension(fname)
+  local filename_parts = rspamd_str_split(fname, '.')
 
+  local ext = {}
+  for n = 1, 2 do
+      ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil
+  end
+  return ext[1],ext[2],filename_parts
 end
 
+local function check_parts_match(task, rule)
+
+  local filter_func = function(p)
+    local content_type,content_subtype = p:get_type()
+    local fname = p:get_filename()
+    local ext, ext2, part_table
+    local extension_check = false
+    local content_type_check = false
+    local text_part_min_words_check = true
+
+    if rule.scan_all_mime_parts == false then
+    -- check file extension and filename regex matching
+      if fname ~= nil then
+        ext,ext2,part_table = gen_extension(fname)
+        lua_util.debugm(rule.module_name, task, '%s: extension found: %s - 2.ext: %s - parts: %s',
+          rule.log_prefix, ext, ext2, part_table)
+        if match_filter(task, ext, rule.mime_parts_filter_ext)
+          or match_filter(task, ext2, rule.mime_parts_filter_ext) then
+          lua_util.debugm(rule.module_name, task, '%s: extension matched: %s', rule.log_prefix, ext)
+          extension_check = true
+        end
+        if match_filter(task, fname, rule.mime_parts_filter_regex) then
+          content_type_check = true
+        end
+      end
+      -- check content type regex matching
+      if content_type ~= nil and content_subtype ~= nil then
+        if match_filter(task, content_type..'/'..content_subtype, rule.mime_parts_filter_regex) then
+          lua_util.debugm(rule.module_name, task, '%s: regex ct: %s', rule.log_prefix,
+            content_type..'/'..content_subtype)
+          content_type_check = true
+        end
+      end
+    end
+
+    -- check text_part has more words than text_part_min_words_check
+    if rule.text_part_min_words and p:is_text() then
+      text_part_min_words_check = p:get_words_count() >= tonumber(rule.text_part_min_words)
+    end
+
+    return (rule.scan_image_mime and p:is_image())
+        or (rule.scan_text_mime and text_part_min_words_check)
+        or (p:is_attachment() and rule.scan_all_mime_parts ~= false)
+        or extension_check
+        or content_type_check
+  end
+
+  return fun.filter(filter_func, task:get_parts())
+end
 
+exports.log_clean = log_clean
 exports.yield_result = yield_result
 exports.match_patterns = match_patterns
 exports.need_av_check = need_av_check
 exports.check_av_cache = check_av_cache
 exports.save_av_cache = save_av_cache
-exports.text_parts_min_words = text_parts_min_words
+exports.create_regex_table = create_regex_table
+exports.check_parts_match = check_parts_match
 
 setmetatable(exports, {
   __call = function(t, override)
@@ -210,4 +325,4 @@ setmetatable(exports, {
   end,
 })
 
-return exports
+return exports

lualib/lua_scanners/dcc.lua

@@ -29,7 +29,7 @@ local fun = require "fun"
 
 local N = 'dcc'
 
-local function dcc_check(task, content, _, rule)
+local function dcc_check(task, content, digest, rule)
   local function dcc_check_uncached ()
     local upstream = rule.upstreams:get_upstream_round_robin()
     local addr = upstream:get_addr()
@@ -81,8 +81,7 @@ local function dcc_check(task, content, _, rule)
 
     local function dcc_callback(err, data, conn)
 
-      if err then
-
+      local function dcc_requery()
         -- set current upstream to fail because an error occurred
         upstream:fail()
 
@@ -91,11 +90,15 @@ local function dcc_check(task, content, _, rule)
 
           retransmits = retransmits - 1
 
+          lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s',
+            rule.log_prefix, err, retransmits)
+
           -- Select a different upstream!
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s: retry IP: %s', rule.log_prefix, addr)
+          lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s',
+            rule.log_prefix, addr, addr:get_port())
 
           tcp.request({
             task = task,
@@ -110,35 +113,35 @@ local function dcc_check(task, content, _, rule)
             fuz2_max = 999999,
           })
         else
-          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
-          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+            'exceed', rule.log_prefix)
+          task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '..
+            'retransmits exceed')
         end
+      end
+
+      if err then
+
+        dcc_requery()
+
       else
         -- Parse the response
         if upstream then upstream:ok() end
         local _,_,result,disposition,header = tostring(data):find("(.-)\n(.-)\n(.-)\n")
-        lua_util.debugm(N, task, 'DCC result=%1 disposition=%2 header="%3"',
+        lua_util.debugm(rule.N, task, 'DCC result=%1 disposition=%2 header="%3"',
             result, disposition, header)
 
         if header then
           local _,_,info = header:find("; (.-)$")
-
           if (result == 'R') then
             -- Reject
             common.yield_result(task, rule, info, rule.default_score)
+            common.save_av_cache(task, digest, rule, info, rule.default_score)
           elseif (result == 'T') then
             -- Temporary failure
             rspamd_logger.warnx(task, 'DCC returned a temporary failure result: %s', result)
-            task:insert_result(rule.symbol_fail,
-                0.0,
-                'tempfail:' .. result)
+            dcc_requery()
           elseif result == 'A' then
-            if rule.log_clean then
-              rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
-                  rule.log_prefix, info)
-            else
-              lua_util.debugm(N, task, '%s: returned result A - info: %s',
-                  rule.log_prefix, info)
 
               local opts = {}
               local score = 0.0
@@ -188,25 +191,36 @@ local function dcc_check(task, content, _, rule)
                 task:insert_result(rule.symbol_bulk,
                     score,
                     opts)
+                common.save_av_cache(task, digest, rule, opts, score)
+              else
+                common.save_av_cache(task, digest, rule, 'OK')
+                if rule.log_clean then
+                  rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
+                      rule.log_prefix, info)
+                else
+                  lua_util.debugm(rule.N, task, '%s: returned result A - info: %s',
+                      rule.log_prefix, info)
               end
             end
           elseif result == 'G' then
             -- do nothing
+            common.save_av_cache(task, digest, rule, 'OK')
             if rule.log_clean then
               rspamd_logger.infox(task, '%s: clean, returned result G - info: %s', rule.log_prefix, info)
             else
-              lua_util.debugm(N, task, '%s: returned result G - info: %s', rule.log_prefix, info)
+              lua_util.debugm(rule.N, task, '%s: returned result G - info: %s', rule.log_prefix, info)
             end
           elseif result == 'S' then
             -- do nothing
+            common.save_av_cache(task, digest, rule, 'OK')
             if rule.log_clean then
               rspamd_logger.infox(task, '%s: clean, returned result S - info: %s', rule.log_prefix, info)
             else
-              lua_util.debugm(N, task, '%s: returned result S - info: %s', rule.log_prefix, info)
+              lua_util.debugm(rule.N, task, '%s: returned result S - info: %s', rule.log_prefix, info)
             end
           else
             -- Unknown result
-            rspamd_logger.warnx(task, 'DCC result error: %1', result);
+            rspamd_logger.warnx(task, '%s: result error: %1', rule.log_prefix, result);
             task:insert_result(rule.symbol_fail,
                 0.0,
                 'error: ' .. result)
@@ -222,21 +236,30 @@ local function dcc_check(task, content, _, rule)
       timeout = rule.timeout or 2.0,
       shutdown = true,
       data = request_data,
-      callback = dcc_callback
+      callback = dcc_callback,
+      body_max = 999999,
+      fuz1_max = 999999,
+      fuz2_max = 999999,
     })
   end
-  if common.need_av_check(task, content, rule, N) then
-    dcc_check_uncached()
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, dcc_check_uncached) then
+      return
+    else
+      dcc_check_uncached()
+    end
   end
 end
 
 local function dcc_config(opts)
 
   local dcc_conf = {
+    N = N,
     default_port = 10045,
     timeout = 5.0,
     log_clean = false,
     retransmits = 2,
+    cache_expire = 7200, -- expire redis in 2h
     message = '${SCANNER}: bulk message found: "${VIRUS}"',
     detection_category = "hash",
     default_score = 1,
@@ -252,8 +275,16 @@ local function dcc_config(opts)
 
   dcc_conf = lua_util.override_defaults(dcc_conf, opts)
 
+  if not dcc_conf.prefix then
+    dcc_conf.prefix = 'rs_' .. dcc_conf.name .. '_'
+  end
+
   if not dcc_conf.log_prefix then
-    dcc_conf.log_prefix = N
+    if dcc_conf.name:lower() == dcc_conf.type:lower() then
+      dcc_conf.log_prefix = dcc_conf.name
+    else
+      dcc_conf.log_prefix = dcc_conf.name .. ' (' .. dcc_conf.type .. ')'
+    end
   end
 
   if not dcc_conf.servers and dcc_conf.socket then
@@ -271,7 +302,7 @@ local function dcc_config(opts)
       dcc_conf.default_port)
 
   if dcc_conf.upstreams then
-    lua_util.add_debug_alias('external_services', N)
+    lua_util.add_debug_alias('external_services', dcc_conf.N)
     return dcc_conf
   end
 
@@ -285,5 +316,5 @@ return {
   description = 'dcc bulk scanner',
   configure = dcc_config,
   check = dcc_check,
-  name = 'dcc'
-}
+  name = N
+}

lualib/lua_scanners/fprot.lua

@@ -31,9 +31,10 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"'
 
 local function fprot_config(opts)
   local fprot_conf = {
-    scan_mime_parts = true;
-    scan_text_mime = false;
-    scan_image_mime = false;
+    N = N,
+    scan_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
     default_port = 10200,
     timeout = 5.0, -- FIXME: this will break task_timeout!
     log_clean = false,
@@ -43,12 +44,18 @@ local function fprot_config(opts)
     message = default_message,
   }
 
-  for k,v in pairs(opts) do
-    fprot_conf[k] = v
-  end
+  fprot_conf = lua_util.override_defaults(fprot_conf, opts)
 
   if not fprot_conf.prefix then
-    fprot_conf.prefix = 'rs_fp'
+    fprot_conf.prefix = 'rs_' .. fprot_conf.name .. '_'
+  end
+
+  if not fprot_conf.log_prefix then
+    if fprot_conf.name:lower() == fprot_conf.type:lower() then
+      fprot_conf.log_prefix = fprot_conf.name
+    else
+      fprot_conf.log_prefix = fprot_conf.name .. ' (' .. fprot_conf.type .. ')'
+    end
   end
 
   if not fprot_conf['servers'] then
@@ -62,7 +69,7 @@ local function fprot_config(opts)
       fprot_conf.default_port)
 
   if fprot_conf['upstreams'] then
-    lua_util.add_debug_alias('antivirus', N)
+    lua_util.add_debug_alias('antivirus', fprot_conf.N)
     return fprot_conf
   end
 
@@ -96,7 +103,7 @@ local function fprot_check(task, content, digest, rule)
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+          lua_util.debugm(rule.N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
 
           tcp.request({
             task = task,
@@ -133,12 +140,12 @@ local function fprot_check(task, content, digest, rule)
           if not vname then
             rspamd_logger.errx(task, 'Unhandled response: %s', data)
           else
-            common.yield_result(task, rule, vname, N)
+            common.yield_result(task, rule, vname)
             cached = vname
           end
         end
         if cached then
-          common.save_av_cache(task, digest, rule, cached, N)
+          common.save_av_cache(task, digest, rule, cached)
         end
       end
     end
@@ -154,8 +161,8 @@ local function fprot_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule, N) then
-    if common.check_av_cache(task, digest, rule, fprot_check_uncached, N) then
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, fprot_check_uncached) then
       return
     else
       fprot_check_uncached()
@@ -168,5 +175,5 @@ return {
   description = 'fprot antivirus',
   configure = fprot_config,
   check = fprot_check,
-  name = 'fprot'
-}
+  name = N
+}

lualib/lua_scanners/icap.lua

@@ -0,0 +1,301 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module icap
+-- This module contains icap access functions.
+-- Currently tested with Symantec, Sophos Savdi, ClamAV/c-icap
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'icap'
+
+local function icap_check(task, content, digest, rule)
+  local function icap_check_uncached ()
+    local upstream = rule.upstreams:get_upstream_round_robin()
+    local addr = upstream:get_addr()
+    local retransmits = rule.retransmits
+    local respond_headers = {}
+
+    -- Build the icap queries
+    local options_request = {
+      "OPTIONS icap://" .. addr:to_string() .. ":" .. addr:get_port() .. "/" .. rule.scheme .. " ICAP/1.0\r\n",
+      "Host:" .. addr:to_string() .. "\r\n",
+      "User-Agent: Rspamd\r\n",
+      "Encapsulated: null-body=0\r\n\r\n",
+    }
+    local size = string.format("%x", tonumber(#content))
+    lua_util.debugm(rule.N, task, '%s: size: %s', rule.log_prefix, size)
+
+    local function get_respond_query()
+      table.insert(respond_headers, 1, 'RESPMOD icap://' .. addr:to_string() .. ':' .. addr:get_port() .. '/'
+        .. rule.scheme .. ' ICAP/1.0\r\n')
+      table.insert(respond_headers, 'Encapsulated: res-body=0\r\n')
+      table.insert(respond_headers, '\r\n')
+      table.insert(respond_headers, size .. '\r\n')
+      table.insert(respond_headers, content)
+      table.insert(respond_headers, '\r\n0\r\n\r\n')
+      return respond_headers
+    end
+
+    local function add_respond_header(name, value)
+      table.insert(respond_headers, name .. ': ' .. value .. '\r\n' )
+    end
+
+    local function icap_result_header_table(result)
+      local icap_headers = {}
+      for s in result:gmatch("[^\r\n]+") do
+        if string.find(s, '^ICAP') then
+          icap_headers['icap'] = s
+        end
+        if string.find(s, '[%a%d-+]-: ') then
+          local _,_,key,value = tostring(s):find("([%a%d-+]-):%s(.+)")
+          icap_headers[key] = value
+        end
+      end
+      lua_util.debugm(rule.N, task, '%s: icap_headers: %s', rule.log_prefix, icap_headers)
+      return icap_headers
+    end
+
+    local function icap_parse_result(icap_headers)
+
+      local threat_string = {}
+
+      --[[
+      @ToDo: handle type in response
+
+      Generic Strings:
+        X-Infection-Found: Type=0; Resolution=2; Threat=Troj/DocDl-OYC;
+        X-Infection-Found: Type=0; Resolution=2; Threat=W97M.Downloader;
+      Symantec String:
+        X-Infection-Found: Type=2; Resolution=2; Threat=Container size violation
+        X-Infection-Found: Type=2; Resolution=2; Threat=Encrypted container violation;
+      Sophos Strings:
+        X-Virus-ID: Troj/DocDl-OYC
+      ]] --
+
+      local pattern_symbols
+      local match
+
+      if icap_headers['X-Infection-Found'] ~= nil then
+        pattern_symbols = "(Type%=%d; .* Threat%=)(.*)([;]+)"
+        match = string.gsub(icap_headers['X-Infection-Found'], pattern_symbols, "%2")
+        lua_util.debugm(rule.N, task, '%s: icap X-Infection-Found: %s', rule.log_prefix, match)
+        table.insert(threat_string, match)
+      elseif icap_headers['X-Virus-ID'] ~= nil then
+        lua_util.debugm(rule.N, task, '%s: icap X-Virus-ID: %s', rule.log_prefix, icap_headers['X-Virus-ID'])
+        table.insert(threat_string, icap_headers['X-Virus-ID'])
+      end
+
+      if #threat_string > 0 then
+        common.yield_result(task, rule, threat_string, rule.default_score)
+        common.save_av_cache(task, digest, rule, threat_string, rule.default_score)
+      else
+        common.save_av_cache(task, digest, rule, 'OK', 0)
+        common.log_clean(task, rule)
+      end
+    end
+
+    local function icap_r_respond_cb(err, data, conn)
+      local result = tostring(data)
+      conn:close()
+
+      local icap_headers = icap_result_header_table(result)
+      -- Find ICAP/1.x 2xx response
+      if string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then
+        icap_parse_result(icap_headers)
+      elseif string.find(icap_headers.icap, 'ICAP%/1%.. [45]%d%d') then
+        -- Find ICAP/1.x 5/4xx response
+        --[[
+        Symantec String:
+          ICAP/1.0 539 Aborted - No AV scanning license
+        SquidClamAV/C-ICAP:
+          ICAP/1.0 500 Server error
+        ]]--
+        rspamd_logger.errx(task, '%s: ICAP ERROR: %s', rule.log_prefix, icap_headers.icap)
+        task:insert_result(rule.symbol_fail, 0.0, icap_headers.icap)
+        return false
+      else
+        rspamd_logger.errx(task, '%s: unhandled response |%s|',
+          rule.log_prefix, string.gsub(result, "\r\n", ", "))
+        task:insert_result(rule.symbol_fail, 0.0, 'unhandled icap response: ' .. icap_headers.icap)
+      end
+    end
+
+    local function icap_w_respond_cb(err, conn)
+      conn:add_read(icap_r_respond_cb, '\r\n\r\n')
+    end
+
+    local function icap_r_options_cb(err, data, conn)
+      local icap_headers = icap_result_header_table(tostring(data))
+
+      if string.find(icap_headers.icap, 'ICAP%/1%.. 2%d%d') then
+        if icap_headers['Methods'] ~= nil and string.find(icap_headers['Methods'], 'RESPMOD') then
+          if icap_headers['Allow'] ~= nil and string.find(icap_headers['Allow'], '204') then
+            add_respond_header('Allow', '204')
+          end
+          conn:add_write(icap_w_respond_cb, get_respond_query())
+        else
+          rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s',
+            rule.log_prefix, icap_headers['Methods'])
+          task:insert_result(rule.symbol_fail, 0.0, 'NO RESPMOD')
+        end
+      else
+        rspamd_logger.errx(task, '%s: OPTIONS query failed: %s',
+          rule.log_prefix, icap_headers.icap)
+        task:insert_result(rule.symbol_fail, 0.0, 'OPTIONS query failed')
+      end
+    end
+
+    local function icap_callback(err, conn)
+
+      local function icap_requery(error)
+        -- set current upstream to fail because an error occurred
+        upstream:fail()
+
+        -- retry with another upstream until retransmits exceeds
+        if retransmits > 0 then
+
+          retransmits = retransmits - 1
+
+          lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s',
+            rule.log_prefix, error, retransmits)
+
+          -- Select a different upstream!
+          upstream = rule.upstreams:get_upstream_round_robin()
+          addr = upstream:get_addr()
+
+          lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s',
+            rule.log_prefix, addr, addr:get_port())
+
+          tcp.request({
+            task = task,
+            host = addr:to_string(),
+            port = addr:get_port(),
+            timeout = rule.timeout,
+            stop_pattern = '\r\n',
+            data = options_request,
+            read = false,
+            callback = icap_callback,
+          })
+        else
+          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+            'exceed - err: %s', rule.log_prefix, error)
+          task:insert_result(rule.symbol_fail, 0.0, 'failed - err: ' .. error)
+        end
+      end
+
+      if err then
+        icap_requery(err)
+      else
+        -- set upstream ok
+        if upstream then upstream:ok() end
+        conn:add_read(icap_r_options_cb, '\r\n\r\n')
+      end
+    end
+
+    tcp.request({
+      task = task,
+      host = addr:to_string(),
+      port = addr:get_port(),
+      timeout = rule.timeout,
+      stop_pattern = '\r\n',
+      data = options_request,
+      read = false,
+      callback = icap_callback,
+    })
+  end
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, icap_check_uncached) then
+      return
+    else
+      icap_check_uncached()
+    end
+  end
+end
+
+
+local function icap_config(opts)
+
+  local icap_conf = {
+    N = N,
+    scan_mime_parts = true,
+    scan_all_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
+    scheme = "scan",
+    default_port = 4020,
+    timeout = 10.0,
+    log_clean = false,
+    retransmits = 2,
+    cache_expire = 7200, -- expire redis in one hour
+    message = '${SCANNER}: threat found with icap scanner: "${VIRUS}"',
+    detection_category = "virus",
+    default_score = 1,
+    action = false,
+  }
+
+  icap_conf = lua_util.override_defaults(icap_conf, opts)
+
+  if not icap_conf.prefix then
+    icap_conf.prefix = 'rs_' .. icap_conf.name .. '_'
+  end
+
+  if not icap_conf.log_prefix then
+    icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')'
+  end
+
+  if not icap_conf.log_prefix then
+    if icap_conf.name:lower() == icap_conf.type:lower() then
+      icap_conf.log_prefix = icap_conf.name
+    else
+      icap_conf.log_prefix = icap_conf.name .. ' (' .. icap_conf.type .. ')'
+    end
+  end
+
+  if not icap_conf.servers then
+    rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+    return nil
+  end
+
+  icap_conf.upstreams = upstream_list.create(rspamd_config,
+    icap_conf.servers,
+    icap_conf.default_port)
+
+  if icap_conf.upstreams then
+    lua_util.add_debug_alias('external_services', icap_conf.N)
+    return icap_conf
+  end
+
+  rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+    icap_conf.servers)
+  return nil
+end
+
+return {
+  type = {N,'virus', 'virus', 'scanner'},
+  description = 'generic icap antivirus',
+  configure = icap_config,
+  check = icap_check,
+  name = N
+}

lualib/lua_scanners/init.lua

@@ -39,6 +39,8 @@ require_scanner('sophos')
 
 -- Other scanners
 require_scanner('dcc')
+require_scanner('oletools')
+require_scanner('icap')
 
 exports.add_scanner = function(name, t, conf_func, check_func)
   assert(type(conf_func) == 'function' and type(check_func) == 'function',
@@ -59,4 +61,4 @@ exports.filter = function(t)
   end, exports))
 end
 
-return exports
+return exports

lualib/lua_scanners/kaspersky_av.lua

@@ -32,9 +32,10 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"'
 
 local function kaspersky_config(opts)
   local kaspersky_conf = {
-    scan_mime_parts = true;
-    scan_text_mime = false;
-    scan_image_mime = false;
+    N = N,
+    scan_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
     product_id = 0,
     log_clean = false,
     timeout = 5.0,
@@ -43,11 +44,22 @@ local function kaspersky_config(opts)
     message = default_message,
     detection_category = "virus",
     tmpdir = '/tmp',
-    prefix = 'rs_ak',
   }
 
   kaspersky_conf = lua_util.override_defaults(kaspersky_conf, opts)
 
+  if not kaspersky_conf.prefix then
+    kaspersky_conf.prefix = 'rs_' .. kaspersky_conf.name .. '_'
+  end
+
+  if not kaspersky_conf.log_prefix then
+    if kaspersky_conf.name:lower() == kaspersky_conf.type:lower() then
+      kaspersky_conf.log_prefix = kaspersky_conf.name
+    else
+      kaspersky_conf.log_prefix = kaspersky_conf.name .. ' (' .. kaspersky_conf.type .. ')'
+    end
+  end
+
   if not kaspersky_conf['servers'] then
     rspamd_logger.errx(rspamd_config, 'no servers defined')
 
@@ -58,7 +70,7 @@ local function kaspersky_config(opts)
       kaspersky_conf['servers'], 0)
 
   if kaspersky_conf['upstreams'] then
-    lua_util.add_debug_alias('antivirus', N)
+    lua_util.add_debug_alias('antivirus', kaspersky_conf.N)
     return kaspersky_conf
   end
 
@@ -110,7 +122,7 @@ local function kaspersky_check(task, content, digest, rule)
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task,
+          lua_util.debugm(rule.N, task,
               '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
 
           tcp.request({
@@ -134,21 +146,15 @@ local function kaspersky_check(task, content, digest, rule)
         upstream:ok()
         data = tostring(data)
         local cached
-        lua_util.debugm(N, task, '%s [%s]: got reply: %s',
+        lua_util.debugm(rule.N, task, '%s [%s]: got reply: %s',
             rule['symbol'], rule['type'], data)
-        if data == 'stream: OK' then
+        if data == 'stream: OK' or data == fname .. ': OK' then
           cached = 'OK'
-          if rule['log_clean'] then
-            rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean',
-                rule['symbol'], rule['type'])
-          else
-            lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean',
-                rule['symbol'], rule['type'])
-          end
+          common.log_clean(task, rule)
         else
           local vname = string.match(data, ': (.+) FOUND')
           if vname then
-            common.yield_result(task, rule, vname, N)
+            common.yield_result(task, rule, vname)
             cached = vname
           else
             rspamd_logger.errx(task, 'unhandled response: %s', data)
@@ -156,7 +162,7 @@ local function kaspersky_check(task, content, digest, rule)
           end
         end
         if cached then
-          common.save_av_cache(task, digest, rule, cached, N)
+          common.save_av_cache(task, digest, rule, cached)
         end
       end
     end
@@ -172,8 +178,8 @@ local function kaspersky_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule, N) then
-    if common.check_av_cache(task, digest, rule, kaspersky_check_uncached, N) then
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, kaspersky_check_uncached) then
       return
     else
       kaspersky_check_uncached()
@@ -186,5 +192,5 @@ return {
   description = 'kaspersky antivirus',
   configure = kaspersky_config,
   check = kaspersky_check,
-  name = 'kaspersky'
-}
+  name = N
+}

lualib/lua_scanners/oletools.lua

@@ -0,0 +1,302 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2018, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module oletools
+-- This module contains oletools access functions.
+-- Olefy is needed: https://github.com/HeinleinSupport/olefy
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local common = require "lua_scanners/common"
+
+local N = 'oletools'
+
+local function oletools_check(task, content, digest, rule)
+  local function oletools_check_uncached ()
+    local upstream = rule.upstreams:get_upstream_round_robin()
+    local addr = upstream:get_addr()
+    local retransmits = rule.retransmits
+    local protocol = 'OLEFY/1.0\nMethod: oletools\nRspamd-ID: ' .. task:get_uid() .. '\n\n'
+
+    local function oletools_callback(err, data)
+
+      local function oletools_requery(error)
+        -- set current upstream to fail because an error occurred
+        upstream:fail()
+
+        -- retry with another upstream until retransmits exceeds
+        if retransmits > 0 then
+
+          retransmits = retransmits - 1
+
+          lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s',
+            rule.log_prefix, error, retransmits)
+
+          -- Select a different upstream!
+          upstream = rule.upstreams:get_upstream_round_robin()
+          addr = upstream:get_addr()
+
+          lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s',
+            rule.log_prefix, addr, addr:get_port())
+
+          tcp.request({
+            task = task,
+            host = addr:to_string(),
+            port = addr:get_port(),
+            timeout = rule.timeout,
+            shutdown = true,
+            data = { protocol, content },
+            callback = oletools_callback,
+          })
+        else
+          rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+            'exceed - err: %s', rule.log_prefix, error)
+          task:insert_result(rule.symbol_fail, 0.0, 'failed - err: ' .. error)
+        end
+      end
+
+      if err then
+
+        oletools_requery(err)
+
+      else
+        -- Parse the response
+        if upstream then upstream:ok() end
+
+        data = tostring(data)
+
+        local ucl_parser = ucl.parser()
+        local ok, ucl_err = ucl_parser:parse_string(tostring(data))
+        if not ok then
+            rspamd_logger.errx(task, "%s: error parsing json response: %s",
+              rule.log_prefix, ucl_err)
+            return
+        end
+
+        local result = ucl_parser:get_object()
+
+        local oletools_rc = {
+          [0] = 'RETURN_OK',
+          [1] = 'RETURN_WARNINGS',
+          [2] = 'RETURN_WRONG_ARGS',
+          [3] = 'RETURN_FILE_NOT_FOUND',
+          [4] = 'RETURN_XGLOB_ERR',
+          [5] = 'RETURN_OPEN_ERROR',
+          [6] = 'RETURN_PARSE_ERROR',
+          [7] = 'RETURN_SEVERAL_ERRS',
+          [8] = 'RETURN_UNEXPECTED',
+          [9] = 'RETURN_ENCRYPTED',
+        }
+
+        if result[1].error ~= nil then
+          rspamd_logger.errx(task, '%s: ERROR found: %s', rule.log_prefix,
+            result[1].error)
+            if result[1].error == 'File too small' then
+              common.save_av_cache(task, digest, rule, 'OK')
+              common.log_clean(task, rule, 'File too small to be scanned for macros')
+            else
+              oletools_requery(result[1].error)
+            end
+        elseif result[3]['return_code'] == 9 then
+          rspamd_logger.warnx(task, '%s: File is encrypted.', rule.log_prefix)
+        elseif result[3]['return_code'] > 6 then
+          rspamd_logger.errx(task, '%s: Error Returned: %s',
+            rule.log_prefix, oletools_rc[result[3]['return_code']])
+          rspamd_logger.errx(task, '%s: Error message: %s',
+            rule.log_prefix, result[2]['message'])
+          task:insert_result(rule.symbol_fail, 0.0, 'failed - err: ' .. oletools_rc[result[3]['return_code']])
+        elseif result[3]['return_code'] > 1 then
+          rspamd_logger.errx(task, '%s: Error message: %s',
+            rule.log_prefix, result[2]['message'])
+          oletools_requery(oletools_rc[result[3]['return_code']])
+        elseif #result[2]['analysis'] == 0 and #result[2]['macros'] == 0 then
+          rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix)
+          task:insert_result(rule.symbol_fail, 0.0, 'oletools unhandled error')
+        elseif result[2]['analysis'] == 'null' and #result[2]['macros'] == 0 then
+          common.save_av_cache(task, digest, rule, 'OK')
+          common.log_clean(task, rule, 'No macro found')
+        elseif #result[2]['macros'] > 0 then
+          -- M=Macros, A=Auto-executable, S=Suspicious keywords, I=IOCs,
+          -- H=Hex strings, B=Base64 strings, D=Dridex strings, V=VBA strings
+          local m_exist = 'M'
+          local m_autoexec = '-'
+          local m_suspicious = '-'
+          local m_iocs = '-'
+          local m_hex = '-'
+          local m_base64 = '-'
+          local m_dridex = '-'
+          local m_vba = '-'
+
+          lua_util.debugm(rule.N, task, '%s: filename: %s', rule.log_prefix, result[2]['file'])
+          lua_util.debugm(rule.N, task, '%s: type: %s', rule.log_prefix, result[2]['type'])
+
+          for _,m in ipairs(result[2]['macros']) do
+            lua_util.debugm(rule.N, task, '%s: macros found - code: %s, ole_stream: %s, '..
+              'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename)
+          end
+
+          local analysis_keyword_table = {}
+
+          for _,a in ipairs(result[2]['analysis']) do
+            lua_util.debugm(rule.N, task, '%s: threat found - type: %s, keyword: %s, '..
+              'description: %s', rule.log_prefix, a.type, a.keyword, a.description)
+            if a.type == 'AutoExec' then
+              m_autoexec = 'A'
+              table.insert(analysis_keyword_table, a.keyword)
+            elseif a.type == 'Suspicious' then
+              m_suspicious = 'S'
+              if a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings'
+              then
+                table.insert(analysis_keyword_table, a.keyword)
+              end
+            elseif a.type == 'IOCs' then
+              m_iocs = 'I'
+            elseif a.type == 'Hex strings' then
+              m_hex = 'H'
+            elseif a.type == 'Base64 strings' then
+              m_base64 = 'B'
+            elseif a.type == 'Dridex strings' then
+              m_dridex = 'D'
+            elseif a.type == 'VBA strings' then
+              m_vba = 'V'
+            end
+          end
+
+         --lua_util.debugm(N, task, '%s: analysis_keyword_table: %s', rule.log_prefix, analysis_keyword_table)
+
+          if rule.extended == false and m_autoexec == 'A' and m_suspicious == 'S' then
+            -- use single string as virus name
+            local threat = 'AutoExec + Suspicious (' .. table.concat(analysis_keyword_table, ',') .. ')'
+            lua_util.debugm(rule.N, task, '%s: threat result: %s', rule.log_prefix, threat)
+            common.yield_result(task, rule, threat, rule.default_score)
+            common.save_av_cache(task, digest, rule, threat, rule.default_score)
+
+          elseif rule.extended == true and #analysis_keyword_table > 0 then
+            -- report any flags (types) and any most keywords as individual virus name
+
+            local flags = m_exist ..
+                          m_autoexec ..
+                          m_suspicious ..
+                          m_iocs ..
+                          m_hex ..
+                          m_base64 ..
+                          m_dridex ..
+                          m_vba
+            table.insert(analysis_keyword_table, 1, flags)
+
+            lua_util.debugm(rule.N, task, '%s: extended threat result: %s',
+              rule.log_prefix, table.concat(analysis_keyword_table, ','))
+
+            common.yield_result(task, rule, analysis_keyword_table, rule.default_score)
+            common.save_av_cache(task, digest, rule, analysis_keyword_table, rule.default_score)
+          else
+            common.save_av_cache(task, digest, rule, 'OK')
+            common.log_clean(task, rule, 'Scanned Macro is OK')
+          end
+
+        else
+          rspamd_logger.warnx(task, '%s: unhandled response', rule.log_prefix)
+          task:insert_result(rule.symbol_fail, 0.0, 'unhandled response')
+        end
+      end
+    end
+
+    tcp.request({
+      task = task,
+      host = addr:to_string(),
+      port = addr:get_port(),
+      timeout = rule.timeout,
+      shutdown = true,
+      data = { protocol, content },
+      callback = oletools_callback,
+    })
+
+  end
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, oletools_check_uncached) then
+      return
+    else
+      oletools_check_uncached()
+    end
+  end
+end
+
+local function oletools_config(opts)
+
+  local oletools_conf = {
+    N = N,
+    scan_mime_parts = false,
+    scan_text_mime = false,
+    scan_image_mime = false,
+    default_port = 5954,
+    timeout = 15.0,
+    log_clean = false,
+    retransmits = 2,
+    cache_expire = 86400, -- expire redis in 1d
+    message = '${SCANNER}: Oletools threat message found: "${VIRUS}"',
+    detection_category = "office macro",
+    default_score = 1,
+    action = false,
+    extended = false,
+  }
+
+  oletools_conf = lua_util.override_defaults(oletools_conf, opts)
+
+  if not oletools_conf.prefix then
+    oletools_conf.prefix = 'rs_' .. oletools_conf.name .. '_'
+  end
+
+  if not oletools_conf.log_prefix then
+    if oletools_conf.name:lower() == oletools_conf.type:lower() then
+      oletools_conf.log_prefix = oletools_conf.name
+    else
+      oletools_conf.log_prefix = oletools_conf.name .. ' (' .. oletools_conf.type .. ')'
+    end
+  end
+
+  if not oletools_conf.servers then
+    rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+    return nil
+  end
+
+  oletools_conf.upstreams = upstream_list.create(rspamd_config,
+    oletools_conf.servers,
+    oletools_conf.default_port)
+
+  if oletools_conf.upstreams then
+    lua_util.add_debug_alias('external_services', oletools_conf.N)
+    return oletools_conf
+  end
+
+  rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+    oletools_conf.servers)
+  return nil
+end
+
+return {
+  type = {N,'attachment scanner', 'hash', 'scanner'},
+  description = 'oletools office macro scanner',
+  configure = oletools_config,
+  check = oletools_check,
+  name = N
+}

lualib/lua_scanners/savapi.lua

@@ -32,9 +32,10 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"'
 
 local function savapi_config(opts)
   local savapi_conf = {
-    scan_mime_parts = true;
-    scan_text_mime = false;
-    scan_image_mime = false;
+    N = N,
+    scan_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
     default_port = 4444, -- note: You must set ListenAddress in savapi.conf
     product_id = 0,
     log_clean = false,
@@ -46,12 +47,18 @@ local function savapi_config(opts)
     tmpdir = '/tmp',
   }
 
-  for k,v in pairs(opts) do
-    savapi_conf[k] = v
-  end
+  savapi_conf = lua_util.override_defaults(savapi_conf, opts)
 
   if not savapi_conf.prefix then
-    savapi_conf.prefix = 'rs_ap'
+    savapi_conf.prefix = 'rs_' .. savapi_conf.name .. '_'
+  end
+
+  if not savapi_conf.log_prefix then
+    if savapi_conf.name:lower() == savapi_conf.type:lower() then
+      savapi_conf.log_prefix = savapi_conf.name
+    else
+      savapi_conf.log_prefix = savapi_conf.name .. ' (' .. savapi_conf.type .. ')'
+    end
   end
 
   if not savapi_conf['servers'] then
@@ -65,7 +72,7 @@ local function savapi_config(opts)
       savapi_conf.default_port)
 
   if savapi_conf['upstreams'] then
-    lua_util.add_debug_alias('antivirus', N)
+    lua_util.add_debug_alias('antivirus', savapi_conf.N)
     return savapi_conf
   end
 
@@ -112,15 +119,15 @@ local function savapi_check(task, content, digest, rule)
       for virus,_ in pairs(vnames) do
         table.insert(vnames_reordered, virus)
       end
-      lua_util.debugm(N, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered)
+      lua_util.debugm(rule.N, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered)
       if #vnames_reordered > 0 then
         local vname = {}
         for _,virus in ipairs(vnames_reordered) do
           table.insert(vname, virus)
         end
 
-        common.yield_result(task, rule, vname, N)
-        common.save_av_cache(task, digest, rule, vname, N)
+        common.yield_result(task, rule, vname)
+        common.save_av_cache(task, digest, rule, vname)
       end
       if conn then
         conn:close()
@@ -129,7 +136,7 @@ local function savapi_check(task, content, digest, rule)
 
     local function savapi_scan2_cb(err, data, conn)
       local result = tostring(data)
-      lua_util.debugm(N, task, "%s: got reply: %s",
+      lua_util.debugm(rule.N, task, "%s: got reply: %s",
           rule['type'], result)
 
       -- Terminal response - clean
@@ -137,7 +144,7 @@ local function savapi_check(task, content, digest, rule)
         if rule['log_clean'] then
           rspamd_logger.infox(task, '%s: message or mime_part is clean', rule['type'])
         end
-        common.save_av_cache(task, digest, rule, 'OK', N)
+        common.save_av_cache(task, digest, rule, 'OK')
         conn:add_write(savapi_fin_cb, 'QUIT\n')
 
         -- Terminal response - infected
@@ -171,7 +178,7 @@ local function savapi_check(task, content, digest, rule)
     local function savapi_greet2_cb(err, data, conn)
       local result = tostring(data)
       if string.find(result, '100 PRODUCT') then
-        lua_util.debugm(N, task, "%s: scanning file: %s",
+        lua_util.debugm(rule.N, task, "%s: scanning file: %s",
             rule['type'], fname)
         conn:add_write(savapi_scan1_cb, {string.format('SCAN %s\n',
             fname)})
@@ -201,7 +208,7 @@ local function savapi_check(task, content, digest, rule)
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+          lua_util.debugm(rule.N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
 
           tcp.request({
             task = task,
@@ -236,8 +243,8 @@ local function savapi_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule, N) then
-    if common.check_av_cache(task, digest, rule, savapi_check_uncached, N) then
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, savapi_check_uncached) then
       return
     else
       savapi_check_uncached()
@@ -250,5 +257,5 @@ return {
   description = 'savapi avira antivirus',
   configure = savapi_config,
   check = savapi_check,
-  name = 'savapi'
-}
+  name = N
+}

lualib/lua_scanners/sophos.lua

@@ -31,9 +31,10 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"'
 
 local function sophos_config(opts)
   local sophos_conf = {
-    scan_mime_parts = true;
-    scan_text_mime = false;
-    scan_image_mime = false;
+    N = N,
+    scan_mime_parts = true,
+    scan_text_mime = false,
+    scan_image_mime = false,
     default_port = 4010,
     timeout = 15.0,
     log_clean = false,
@@ -45,12 +46,18 @@ local function sophos_config(opts)
     savdi_report_oversize = false,
   }
 
-  for k,v in pairs(opts) do
-    sophos_conf[k] = v
-  end
+  sophos_conf = lua_util.override_defaults(sophos_conf, opts)
 
   if not sophos_conf.prefix then
-    sophos_conf.prefix = 'rs_sp'
+    sophos_conf.prefix = 'rs_' .. sophos_conf.name .. '_'
+  end
+
+  if not sophos_conf.log_prefix then
+    if sophos_conf.name:lower() == sophos_conf.type:lower() then
+      sophos_conf.log_prefix = sophos_conf.name
+    else
+      sophos_conf.log_prefix = sophos_conf.name .. ' (' .. sophos_conf.type .. ')'
+    end
   end
 
   if not sophos_conf['servers'] then
@@ -64,7 +71,7 @@ local function sophos_config(opts)
       sophos_conf.default_port)
 
   if sophos_conf['upstreams'] then
-    lua_util.add_debug_alias('antivirus', N)
+    lua_util.add_debug_alias('antivirus', sophos_conf.N)
     return sophos_conf
   end
 
@@ -97,7 +104,7 @@ local function sophos_check(task, content, digest, rule)
           upstream = rule.upstreams:get_upstream_round_robin()
           addr = upstream:get_addr()
 
-          lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+          lua_util.debugm(rule.N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
 
           tcp.request({
             task = task,
@@ -114,19 +121,19 @@ local function sophos_check(task, content, digest, rule)
       else
         upstream:ok()
         data = tostring(data)
-        lua_util.debugm(N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data)
+        lua_util.debugm(rule.N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data)
         local vname = string.match(data, 'VIRUS (%S+) ')
         if vname then
-          common.yield_result(task, rule, vname, N)
-          common.save_av_cache(task, digest, rule, vname, N)
+          common.yield_result(task, rule, vname)
+          common.save_av_cache(task, digest, rule, vname)
         else
           if string.find(data, 'DONE OK') then
             if rule['log_clean'] then
-              rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+              rspamd_logger.infox(task, '%s: message or mime_part is clean', rule.log_prefix)
             else
-              lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+              lua_util.debugm(rule.N, task, '%s: message or mime_part is clean', rule.log_prefix)
             end
-            common.save_av_cache(task, digest, rule, 'OK', N)
+            common.save_av_cache(task, digest, rule, 'OK')
             -- not finished - continue
           elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then
             conn:add_read(sophos_callback)
@@ -134,15 +141,15 @@ local function sophos_check(task, content, digest, rule)
           elseif string.find(data, 'FAIL 0212') then
             rspamd_logger.infox(task, 'Message is ENCRYPTED (0212 SOPHOS_SAVI_ERROR_FILE_ENCRYPTED): %s', data)
             if rule['savdi_report_encrypted'] then
-              common.yield_result(task, rule, "SAVDI_FILE_ENCRYPTED", N)
-              common.save_av_cache(task, digest, rule, "SAVDI_FILE_ENCRYPTED", N)
+              common.yield_result(task, rule, "SAVDI_FILE_ENCRYPTED")
+              common.save_av_cache(task, digest, rule, "SAVDI_FILE_ENCRYPTED")
             end
             -- set pseudo virus if configured, else set fail since part was not scanned
           elseif string.find(data, 'REJ 4') then
             if rule['savdi_report_oversize'] then
               rspamd_logger.infox(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data)
-              common.yield_result(task, rule, "SAVDI_FILE_OVERSIZED", N)
-              common.save_av_cache(task, digest, rule, "SAVDI_FILE_OVERSIZED", N)
+              common.yield_result(task, rule, "SAVDI_FILE_OVERSIZED")
+              common.save_av_cache(task, digest, rule, "SAVDI_FILE_OVERSIZED")
             else
               rspamd_logger.errx(task, 'SAVDI: Message is OVERSIZED (SSSP reject code 4): %s', data)
               task:insert_result(rule['symbol_fail'], 0.0, 'Message is OVERSIZED (SSSP reject code 4):' .. data)
@@ -170,8 +177,8 @@ local function sophos_check(task, content, digest, rule)
     })
   end
 
-  if common.need_av_check(task, content, rule, N) then
-    if common.check_av_cache(task, digest, rule, sophos_check_uncached, N) then
+  if common.need_av_check(task, content, rule) then
+    if common.check_av_cache(task, digest, rule, sophos_check_uncached) then
       return
     else
       sophos_check_uncached()
@@ -184,5 +191,5 @@ return {
   description = 'sophos antivirus',
   configure = sophos_config,
   check = sophos_check,
-  name = 'sophos'
-}
+  name = N
+}

src/plugins/lua/antivirus.lua

@@ -15,10 +15,10 @@ limitations under the License.
 ]] --
 
 local rspamd_logger = require "rspamd_logger"
-local rspamd_regexp = require "rspamd_regexp"
 local lua_util = require "lua_util"
 local fun = require "fun"
 local lua_antivirus = require("lua_scanners").filter('antivirus')
+local common = require "lua_scanners/common"
 local redis_params
 
 local N = "antivirus"
@@ -70,29 +70,29 @@ end
 
 
 local function add_antivirus_rule(sym, opts)
-  if not opts['type'] then
+  if not opts.type then
     rspamd_logger.errx(rspamd_config, 'unknown type for AV rule %s', sym)
     return nil
   end
 
-  if not opts['symbol'] then opts['symbol'] = sym:upper() end
-  local cfg = lua_antivirus[opts['type']]
+  if not opts.symbol then opts.symbol = sym:upper() end
+  local cfg = lua_antivirus[opts.type]
 
   if not cfg then
     rspamd_logger.errx(rspamd_config, 'unknown antivirus type: %s',
-        opts['type'])
+        opts.type)
     return nil
   end
 
-  if not opts['symbol_fail'] then
-    opts['symbol_fail'] = string.upper(opts['type']) .. '_FAIL'
+  if not opts.symbol_fail then
+    opts.symbol_fail = opts.symbol .. '_FAIL'
   end
 
   -- WORKAROUND for deprecated attachments_only
-  if opts['attachments_only'] ~= nil then
-    opts['scan_mime_parts'] = opts['attachments_only']
+  if opts.attachments_only ~= nil then
+    opts.scan_mime_parts = opts.attachments_only
     rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. '..
-     'Please use scan_mime_parts = %s instead', opts['symbol'], opts['type'], opts['attachments_only'])
+     'Please use scan_mime_parts = %s instead', opts.symbol, opts.type, opts.attachments_only)
   end
   -- WORKAROUND for deprecated attachments_only
 
@@ -103,52 +103,25 @@ local function add_antivirus_rule(sym, opts)
 
   if not rule then
     rspamd_logger.errx(rspamd_config, 'cannot configure %s for %s',
-      opts['type'], opts['symbol'])
+      opts.type, opts.symbol)
     return nil
   end
 
-  if type(opts['patterns']) == 'table' then
-    rule['patterns'] = {}
-    if opts['patterns'][1] then
-      for i, p in ipairs(opts['patterns']) do
-        if type(p) == 'table' then
-          local new_set = {}
-          for k, v in pairs(p) do
-            new_set[k] = rspamd_regexp.create_cached(v)
-          end
-          rule['patterns'][i] = new_set
-        else
-          rule['patterns'][i] = {}
-        end
-      end
-    else
-      for k, v in pairs(opts['patterns']) do
-        rule['patterns'][k] = rspamd_regexp.create_cached(v)
-      end
-    end
-  end
+  rule.patterns = common.create_regex_table(opts.patterns or {})
 
-  if opts['whitelist'] then
-    rule['whitelist'] = rspamd_config:add_hash_map(opts['whitelist'])
+  if opts.whitelist then
+    rule.whitelist = rspamd_config:add_hash_map(opts.whitelist)
   end
 
   return function(task)
     if rule.scan_mime_parts then
-      local parts = task:get_parts() or {}
-
-      local filter_func = function(p)
-        return (rule.scan_image_mime and p:is_image())
-            or (rule.scan_text_mime and p:is_text())
-            or (p:is_attachment())
-      end
 
       fun.each(function(p)
         local content = p:get_content()
-
         if content and #content > 0 then
           cfg.check(task, content, p:get_digest(), rule)
         end
-      end, fun.filter(filter_func, parts))
+      end, common.check_parts_match(task, rule))
 
     else
       cfg.check(task, task:get_content(), task:get_digest(), rule)
@@ -164,6 +137,7 @@ if opts and type(opts) == 'table' then
   for k, m in pairs(opts) do
     if type(m) == 'table' and m.servers then
       if not m.type then m.type = k end
+      if not m.name then m.name = k end
       local cb = add_antivirus_rule(k, m)
 
       if not cb then

src/plugins/lua/external_services.lua

@@ -1,5 +1,6 @@
 --[[
 Copyright (c) 2019, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,9 +16,10 @@ limitations under the License.
 ]] --
 
 local rspamd_logger = require "rspamd_logger"
-local rspamd_regexp = require "rspamd_regexp"
 local lua_util = require "lua_util"
+local fun = require "fun"
 local lua_scanners = require("lua_scanners").filter('scanner')
+local common = require "lua_scanners/common"
 local redis_params
 
 local N = "external_services"
@@ -28,13 +30,72 @@ if confighelp then
     [[
 external_services {
   # multiple scanners could be checked, for each we create a configuration block with an arbitrary name
+
+  oletools {
+    # If set force this action if any virus is found (default unset: no action is forced)
+    # action = "reject";
+    # If set, then rejection message is set to this value (mention single quotes)
+    # If `max_size` is set, messages > n bytes in size are not scanned
+    # max_size = 20000000;
+    # log_clean = true;
+    # servers = "127.0.0.1:10050";
+    # cache_expire = 86400;
+    # scan_mime_parts = true;
+    # extended = false;
+    # if `patterns` is specified virus name will be matched against provided regexes and the related
+    # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+    patterns {
+      # symbol_name = "pattern";
+      JUST_EICAR = "^Eicar-Test-Signature$";
+    }
+    # mime-part regex matching in content-type or filename
+    mime_parts_filter_regex {
+      #GEN1 = "application\/octet-stream";
+      DOC2 = "application\/msword";
+      DOC3 = "application\/vnd\.ms-word.*";
+      XLS = "application\/vnd\.ms-excel.*";
+      PPT = "application\/vnd\.ms-powerpoint.*";
+      GEN2 = "application\/vnd\.openxmlformats-officedocument.*";
+    }
+    # Mime-Part filename extension matching (no regex)
+    mime_parts_filter_ext {
+      doc = "doc";
+      dot = "dot";
+      docx = "docx";
+      dotx = "dotx";
+      docm = "docm";
+      dotm = "dotm";
+      xls = "xls";
+      xlt = "xlt";
+      xla = "xla";
+      xlsx = "xlsx";
+      xltx = "xltx";
+      xlsm = "xlsm";
+      xltm = "xltm";
+      xlam = "xlam";
+      xlsb = "xlsb";
+      ppt = "ppt";
+      pot = "pot";
+      pps = "pps";
+      ppa = "ppa";
+      pptx = "pptx";
+      potx = "potx";
+      ppsx = "ppsx";
+      ppam = "ppam";
+      pptm = "pptm";
+      potm = "potm";
+      ppsm = "ppsm";
+    }
+    # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+    whitelist = "/etc/rspamd/antivirus.wl";
+  }
   dcc {
     # If set force this action if any virus is found (default unset: no action is forced)
     # action = "reject";
     # If set, then rejection message is set to this value (mention single quotes)
     # If `max_size` is set, messages > n bytes in size are not scanned
     max_size = 20000000;
-    servers = "127.0.0.1:3310";
+    #servers = "127.0.0.1:10045;
     # if `patterns` is specified virus name will be matched against provided regexes and the related
     # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
     patterns {
@@ -51,22 +112,22 @@ end
 
 
 local function add_scanner_rule(sym, opts)
-  if not opts['type'] then
+  if not opts.type then
     rspamd_logger.errx(rspamd_config, 'unknown type for external scanner rule %s', sym)
     return nil
   end
 
-  if not opts['symbol'] then opts['symbol'] = sym:upper() end
-  local cfg = lua_scanners[opts['type']]
+  if not opts.symbol then opts.symbol = sym:upper() end
+  local cfg = lua_scanners[opts.type]
 
   if not cfg then
-    rspamd_logger.errx(rspamd_config, 'unknown antivirus type: %s',
-        opts['type'])
+    rspamd_logger.errx(rspamd_config, 'unknown external scanner type: %s',
+        opts.type)
     return nil
   end
 
-  if not opts['symbol_fail'] then
-    opts['symbol_fail'] = string.upper(opts['type']) .. '_FAIL'
+  if not opts.symbol_fail then
+    opts.symbol_fail = opts.symbol .. '_FAIL'
   end
 
   local rule = cfg.configure(opts)
@@ -76,37 +137,39 @@ local function add_scanner_rule(sym, opts)
 
   if not rule then
     rspamd_logger.errx(rspamd_config, 'cannot configure %s for %s',
-      opts['type'], opts['symbol'])
+      opts.type, opts.symbol)
     return nil
   end
 
-  if type(opts['patterns']) == 'table' then
-    rule['patterns'] = {}
-    if opts['patterns'][1] then
-      for i, p in ipairs(opts['patterns']) do
-        if type(p) == 'table' then
-          local new_set = {}
-          for k, v in pairs(p) do
-            new_set[k] = rspamd_regexp.create_cached(v)
-          end
-          rule['patterns'][i] = new_set
-        else
-          rule['patterns'][i] = {}
-        end
-      end
-    else
-      for k, v in pairs(opts['patterns']) do
-        rule['patterns'][k] = rspamd_regexp.create_cached(v)
-      end
-    end
+  -- if any mime_part filter defined, do not scan all attachments
+  if opts.mime_parts_filter_regex ~= nil
+    or opts.mime_parts_filter_ext ~= nil then
+      rule.scan_all_mime_parts = false
   end
 
-  if opts['whitelist'] then
-    rule['whitelist'] = rspamd_config:add_hash_map(opts['whitelist'])
+  rule.patterns = common.create_regex_table(opts.patterns or {})
+
+  rule.mime_parts_filter_regex = common.create_regex_table(opts.mime_parts_filter_regex or {})
+
+  rule.mime_parts_filter_ext = common.create_regex_table(opts.mime_parts_filter_ext or {})
+
+  if opts.whitelist then
+    rule.whitelist = rspamd_config:add_hash_map(opts.whitelist)
   end
 
   return function(task)
-    cfg.check(task, task:get_content(), task:get_digest(), rule)
+    if rule.scan_mime_parts then
+
+      fun.each(function(p)
+        local content = p:get_content()
+        if content and #content > 0 then
+          cfg.check(task, content, p:get_digest(), rule)
+        end
+      end, common.check_parts_match(task, rule))
+
+    else
+      cfg.check(task, task:get_content(), task:get_digest(), rule)
+    end
   end
 end
 
@@ -118,6 +181,7 @@ if opts and type(opts) == 'table' then
   for k, m in pairs(opts) do
     if type(m) == 'table' and m.servers then
       if not m.type then m.type = k end
+      if not m.name then m.name = k end
       local cb = add_scanner_rule(k, m)
 
       if not cb then