aboutsummaryrefslogtreecommitdiffstats
path: root/lualib/lua_scanners/common.lua
diff options
context:
space:
mode:
Diffstat (limited to 'lualib/lua_scanners/common.lua')
-rw-r--r--lualib/lua_scanners/common.lua193
1 files changed, 154 insertions, 39 deletions
diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua
index da1a4dd49..0c76004eb 100644
--- a/lualib/lua_scanners/common.lua
+++ b/lualib/lua_scanners/common.lua
@@ -1,5 +1,6 @@
--[[
Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -20,30 +21,43 @@ limitations under the License.
--]]
local rspamd_logger = require "rspamd_logger"
+local rspamd_regexp = require "rspamd_regexp"
local lua_util = require "lua_util"
local lua_redis = require "lua_redis"
local fun = require "fun"
local exports = {}
-local function match_patterns(default_sym, found, patterns)
- if type(patterns) ~= 'table' then return default_sym end
+local function log_clean(task, rule, msg)
+
+ msg = msg or 'message or mime_part is clean'
+
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: %s', rule.log_prefix, msg)
+ else
+ lua_util.debugm(rule.module_name, task, '%s: %s', rule.log_prefix, msg)
+ end
+
+end
+
+local function match_patterns(default_sym, found, patterns, dyn_weight)
+ if type(patterns) ~= 'table' then return default_sym, dyn_weight end
if not patterns[1] then
for sym, pat in pairs(patterns) do
if pat:match(found) then
- return sym
+ return sym, '1'
end
end
- return default_sym
+ return default_sym, dyn_weight
else
for _, p in ipairs(patterns) do
for sym, pat in pairs(p) do
if pat:match(found) then
- return sym
+ return sym, '1'
end
end
end
- return default_sym
+ return default_sym, dyn_weight
end
end
@@ -51,23 +65,23 @@ local function yield_result(task, rule, vname, N, dyn_weight)
local all_whitelisted = true
if not dyn_weight then dyn_weight = 1.0 end
if type(vname) == 'string' then
- local symname = match_patterns(rule.symbol, vname, rule.patterns)
+ local symname, symscore = match_patterns(rule.symbol, vname, rule.patterns, dyn_weight)
if rule.whitelist and rule.whitelist:get_key(vname) then
- rspamd_logger.infox(task, '%s: "%s" is in whitelist', N, vname)
+ rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vname)
return
end
- task:insert_result(symname, 1.0, vname)
- rspamd_logger.infox(task, '%s: %s found: "%s"', N, rule.detection_category, vname)
+ task:insert_result(symname, symscore, vname)
+ rspamd_logger.infox(task, '%s: %s found: "%s"', rule.log_prefix, rule.detection_category, vname)
elseif type(vname) == 'table' then
for _, vn in ipairs(vname) do
- local symname = match_patterns(rule.symbol, vn, rule.patterns)
+ local symname, symscore = match_patterns(rule.symbol, vn, rule.patterns, dyn_weight)
if rule.whitelist and rule.whitelist:get_key(vn) then
- rspamd_logger.infox(task, '%s: "%s" is in whitelist', N, vn)
+ rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vn)
else
all_whitelisted = false
- task:insert_result(symname, dyn_weight, vn)
+ task:insert_result(symname, symscore, vn)
rspamd_logger.infox(task, '%s: %s found: "%s"',
- N, rule.detection_category, vn)
+ rule.log_prefix, rule.detection_category, vn)
end
end
end
@@ -76,43 +90,45 @@ local function yield_result(task, rule, vname, N, dyn_weight)
if all_whitelisted then return end
vname = table.concat(vname, '; ')
end
- task:set_pre_result(rule['action'],
+ task:set_pre_result(rule.action,
lua_util.template(rule.message or 'Rejected', {
- SCANNER = N,
+ SCANNER = rule.name,
VIRUS = vname,
}), N)
end
end
-local function message_not_too_large(task, content, rule, N)
+local function message_not_too_large(task, content, rule)
local max_size = tonumber(rule.max_size)
if not max_size then return true end
if #content > max_size then
rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)",
- N, #content, max_size)
+ rule.log_prefix, #content, max_size)
return false
end
return true
end
-local function need_av_check(task, content, rule, N)
- return message_not_too_large(task, content, rule, N)
+local function need_av_check(task, content, rule)
+ return message_not_too_large(task, content, rule)
end
-local function check_av_cache(task, digest, rule, fn, N)
+local function check_av_cache(task, digest, rule, fn)
local key = digest
local function redis_av_cb(err, data)
if data and type(data) == 'string' then
-- Cached
- if data ~= 'OK' then
- lua_util.debugm(N, task, 'got cached result for %s: %s',
- key, data)
- data = lua_util.str_split(data, '\v')
- yield_result(task, rule, data, N)
+ data = rspamd_str_split(data, '\t')
+ local threat_string = rspamd_str_split(data[1], '\v')
+ local score = data[2] or rule.default_score
+ if threat_string[1] ~= 'OK' then
+ lua_util.debugm(rule.module_name, task, '%s: got cached threat result for %s: %s',
+ rule.log_prefix, key, threat_string[1])
+ yield_result(task, rule, threat_string, score)
else
- lua_util.debugm(N, task, 'got cached result for %s: %s',
- key, data)
+ lua_util.debugm(rule.module_name, task, '%s: got cached negative result for %s: %s',
+ rule.log_prefix, key, threat_string[1])
end
else
if err then
@@ -124,7 +140,7 @@ local function check_av_cache(task, digest, rule, fn, N)
if rule.redis_params then
- key = rule['prefix'] .. key
+ key = rule.prefix .. key
if lua_redis.redis_make_request(task,
rule.redis_params, -- connect params
@@ -141,7 +157,7 @@ local function check_av_cache(task, digest, rule, fn, N)
return false
end
-local function save_av_cache(task, digest, rule, to_save, N)
+local function save_av_cache(task, digest, rule, to_save, dyn_weight)
local key = digest
local function redis_set_cb(err)
@@ -150,8 +166,7 @@ local function save_av_cache(task, digest, rule, to_save, N)
rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s',
rule.detection_category, to_save, key, err)
else
- lua_util.debugm(N, task, 'saved cached result for %s: %s',
- key, to_save)
+ lua_util.debugm(rule.module_name, task, '%s: saved cached result for %s: %s', rule.log_prefix, key, to_save)
end
end
@@ -159,6 +174,8 @@ local function save_av_cache(task, digest, rule, to_save, N)
to_save = table.concat(to_save, '\v')
end
+ local value = table.concat({to_save, dyn_weight}, '\t')
+
if rule.redis_params and rule.prefix then
key = rule.prefix .. key
@@ -168,29 +185,127 @@ local function save_av_cache(task, digest, rule, to_save, N)
true, -- is write
redis_set_cb, --callback
'SETEX', -- command
- { key, rule.cache_expire or 0, to_save }
+ { key, rule.cache_expire or 0, value }
)
end
return false
end
-local function text_parts_min_words(task, min_words)
- local filter_func = function(p)
- return p:get_words_count() >= min_words
+local function create_regex_table(patterns)
+ local regex_table = {}
+ if patterns[1] then
+ for i, p in ipairs(patterns) do
+ if type(p) == 'table' then
+ local new_set = {}
+ for k, v in pairs(p) do
+ new_set[k] = rspamd_regexp.create_cached(v)
+ end
+ regex_table[i] = new_set
+ else
+ regex_table[i] = {}
+ end
+ end
+ else
+ for k, v in pairs(patterns) do
+ regex_table[k] = rspamd_regexp.create_cached(v)
+ end
+ end
+ return regex_table
+end
+
+local function match_filter(task, found, patterns)
+ if type(patterns) ~= 'table' then return false end
+ if not patterns[1] then
+ for _, pat in pairs(patterns) do
+ if pat:match(found) then
+ return true
+ end
+ end
+ return false
+ else
+ for _, p in ipairs(patterns) do
+ for _, pat in ipairs(p) do
+ if pat:match(found) then
+ return true
+ end
+ end
+ end
+ return false
end
+end
- return fun.any(filter_func, task:get_text_parts())
+-- borrowed from mime_types.lua
+-- ext is the last extension, LOWERCASED
+-- ext2 is the one before last extension LOWERCASED
+local function gen_extension(fname)
+ local filename_parts = rspamd_str_split(fname, '.')
+ local ext = {}
+ for n = 1, 2 do
+ ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil
+ end
+ return ext[1],ext[2],filename_parts
end
+local function check_parts_match(task, rule)
+
+ local filter_func = function(p)
+ local content_type,content_subtype = p:get_type()
+ local fname = p:get_filename()
+ local ext, ext2, part_table
+ local extension_check = false
+ local content_type_check = false
+ local text_part_min_words_check = true
+
+ if rule.scan_all_mime_parts == false then
+ -- check file extension and filename regex matching
+ if fname ~= nil then
+ ext,ext2,part_table = gen_extension(fname)
+ lua_util.debugm(rule.module_name, task, '%s: extension found: %s - 2.ext: %s - parts: %s',
+ rule.log_prefix, ext, ext2, part_table)
+ if match_filter(task, ext, rule.mime_parts_filter_ext)
+ or match_filter(task, ext2, rule.mime_parts_filter_ext) then
+ lua_util.debugm(rule.module_name, task, '%s: extension matched: %s', rule.log_prefix, ext)
+ extension_check = true
+ end
+ if match_filter(task, fname, rule.mime_parts_filter_regex) then
+ content_type_check = true
+ end
+ end
+ -- check content type regex matching
+ if content_type ~= nil and content_subtype ~= nil then
+ if match_filter(task, content_type..'/'..content_subtype, rule.mime_parts_filter_regex) then
+ lua_util.debugm(rule.module_name, task, '%s: regex ct: %s', rule.log_prefix,
+ content_type..'/'..content_subtype)
+ content_type_check = true
+ end
+ end
+ end
+
+ -- check text_part has more words than text_part_min_words_check
+ if rule.text_part_min_words and p:is_text() then
+ text_part_min_words_check = p:get_words_count() >= tonumber(rule.text_part_min_words)
+ end
+
+ return (rule.scan_image_mime and p:is_image())
+ or (rule.scan_text_mime and text_part_min_words_check)
+ or (p:is_attachment() and rule.scan_all_mime_parts ~= false)
+ or extension_check
+ or content_type_check
+ end
+
+ return fun.filter(filter_func, task:get_parts())
+end
+exports.log_clean = log_clean
exports.yield_result = yield_result
exports.match_patterns = match_patterns
exports.need_av_check = need_av_check
exports.check_av_cache = check_av_cache
exports.save_av_cache = save_av_cache
-exports.text_parts_min_words = text_parts_min_words
+exports.create_regex_table = create_regex_table
+exports.check_parts_match = check_parts_match
setmetatable(exports, {
__call = function(t, override)
@@ -210,4 +325,4 @@ setmetatable(exports, {
end,
})
-return exports \ No newline at end of file
+return exports