diff options
Diffstat (limited to 'lualib/lua_scanners/common.lua')
-rw-r--r-- | lualib/lua_scanners/common.lua | 193 |
1 files changed, 154 insertions, 39 deletions
diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua index da1a4dd49..0c76004eb 100644 --- a/lualib/lua_scanners/common.lua +++ b/lualib/lua_scanners/common.lua @@ -1,5 +1,6 @@ --[[ Copyright (c) 2018, Vsevolod Stakhov <vsevolod@highsecure.ru> +Copyright (c) 2019, Carsten Rosenberg <c.rosenberg@heinlein-support.de> Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,30 +21,43 @@ limitations under the License. --]] local rspamd_logger = require "rspamd_logger" +local rspamd_regexp = require "rspamd_regexp" local lua_util = require "lua_util" local lua_redis = require "lua_redis" local fun = require "fun" local exports = {} -local function match_patterns(default_sym, found, patterns) - if type(patterns) ~= 'table' then return default_sym end +local function log_clean(task, rule, msg) + + msg = msg or 'message or mime_part is clean' + + if rule.log_clean then + rspamd_logger.infox(task, '%s: %s', rule.log_prefix, msg) + else + lua_util.debugm(rule.module_name, task, '%s: %s', rule.log_prefix, msg) + end + +end + +local function match_patterns(default_sym, found, patterns, dyn_weight) + if type(patterns) ~= 'table' then return default_sym, dyn_weight end if not patterns[1] then for sym, pat in pairs(patterns) do if pat:match(found) then - return sym + return sym, '1' end end - return default_sym + return default_sym, dyn_weight else for _, p in ipairs(patterns) do for sym, pat in pairs(p) do if pat:match(found) then - return sym + return sym, '1' end end end - return default_sym + return default_sym, dyn_weight end end @@ -51,23 +65,23 @@ local function yield_result(task, rule, vname, N, dyn_weight) local all_whitelisted = true if not dyn_weight then dyn_weight = 1.0 end if type(vname) == 'string' then - local symname = match_patterns(rule.symbol, vname, rule.patterns) + local symname, symscore = match_patterns(rule.symbol, vname, rule.patterns, dyn_weight) if rule.whitelist and rule.whitelist:get_key(vname) then - rspamd_logger.infox(task, '%s: "%s" is in whitelist', N, vname) + rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vname) return end - task:insert_result(symname, 1.0, vname) - rspamd_logger.infox(task, '%s: %s found: "%s"', N, rule.detection_category, vname) + task:insert_result(symname, symscore, vname) + rspamd_logger.infox(task, '%s: %s found: "%s"', rule.log_prefix, rule.detection_category, vname) elseif type(vname) == 'table' then for _, vn in ipairs(vname) do - local symname = match_patterns(rule.symbol, vn, rule.patterns) + local symname, symscore = match_patterns(rule.symbol, vn, rule.patterns, dyn_weight) if rule.whitelist and rule.whitelist:get_key(vn) then - rspamd_logger.infox(task, '%s: "%s" is in whitelist', N, vn) + rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vn) else all_whitelisted = false - task:insert_result(symname, dyn_weight, vn) + task:insert_result(symname, symscore, vn) rspamd_logger.infox(task, '%s: %s found: "%s"', - N, rule.detection_category, vn) + rule.log_prefix, rule.detection_category, vn) end end end @@ -76,43 +90,45 @@ local function yield_result(task, rule, vname, N, dyn_weight) if all_whitelisted then return end vname = table.concat(vname, '; ') end - task:set_pre_result(rule['action'], + task:set_pre_result(rule.action, lua_util.template(rule.message or 'Rejected', { - SCANNER = N, + SCANNER = rule.name, VIRUS = vname, }), N) end end -local function message_not_too_large(task, content, rule, N) +local function message_not_too_large(task, content, rule) local max_size = tonumber(rule.max_size) if not max_size then return true end if #content > max_size then rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)", - N, #content, max_size) + rule.log_prefix, #content, max_size) return false end return true end -local function need_av_check(task, content, rule, N) - return message_not_too_large(task, content, rule, N) +local function need_av_check(task, content, rule) + return message_not_too_large(task, content, rule) end -local function check_av_cache(task, digest, rule, fn, N) +local function check_av_cache(task, digest, rule, fn) local key = digest local function redis_av_cb(err, data) if data and type(data) == 'string' then -- Cached - if data ~= 'OK' then - lua_util.debugm(N, task, 'got cached result for %s: %s', - key, data) - data = lua_util.str_split(data, '\v') - yield_result(task, rule, data, N) + data = rspamd_str_split(data, '\t') + local threat_string = rspamd_str_split(data[1], '\v') + local score = data[2] or rule.default_score + if threat_string[1] ~= 'OK' then + lua_util.debugm(rule.module_name, task, '%s: got cached threat result for %s: %s', + rule.log_prefix, key, threat_string[1]) + yield_result(task, rule, threat_string, score) else - lua_util.debugm(N, task, 'got cached result for %s: %s', - key, data) + lua_util.debugm(rule.module_name, task, '%s: got cached negative result for %s: %s', + rule.log_prefix, key, threat_string[1]) end else if err then @@ -124,7 +140,7 @@ local function check_av_cache(task, digest, rule, fn, N) if rule.redis_params then - key = rule['prefix'] .. key + key = rule.prefix .. key if lua_redis.redis_make_request(task, rule.redis_params, -- connect params @@ -141,7 +157,7 @@ local function check_av_cache(task, digest, rule, fn, N) return false end -local function save_av_cache(task, digest, rule, to_save, N) +local function save_av_cache(task, digest, rule, to_save, dyn_weight) local key = digest local function redis_set_cb(err) @@ -150,8 +166,7 @@ local function save_av_cache(task, digest, rule, to_save, N) rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s', rule.detection_category, to_save, key, err) else - lua_util.debugm(N, task, 'saved cached result for %s: %s', - key, to_save) + lua_util.debugm(rule.module_name, task, '%s: saved cached result for %s: %s', rule.log_prefix, key, to_save) end end @@ -159,6 +174,8 @@ local function save_av_cache(task, digest, rule, to_save, N) to_save = table.concat(to_save, '\v') end + local value = table.concat({to_save, dyn_weight}, '\t') + if rule.redis_params and rule.prefix then key = rule.prefix .. key @@ -168,29 +185,127 @@ local function save_av_cache(task, digest, rule, to_save, N) true, -- is write redis_set_cb, --callback 'SETEX', -- command - { key, rule.cache_expire or 0, to_save } + { key, rule.cache_expire or 0, value } ) end return false end -local function text_parts_min_words(task, min_words) - local filter_func = function(p) - return p:get_words_count() >= min_words +local function create_regex_table(patterns) + local regex_table = {} + if patterns[1] then + for i, p in ipairs(patterns) do + if type(p) == 'table' then + local new_set = {} + for k, v in pairs(p) do + new_set[k] = rspamd_regexp.create_cached(v) + end + regex_table[i] = new_set + else + regex_table[i] = {} + end + end + else + for k, v in pairs(patterns) do + regex_table[k] = rspamd_regexp.create_cached(v) + end + end + return regex_table +end + +local function match_filter(task, found, patterns) + if type(patterns) ~= 'table' then return false end + if not patterns[1] then + for _, pat in pairs(patterns) do + if pat:match(found) then + return true + end + end + return false + else + for _, p in ipairs(patterns) do + for _, pat in ipairs(p) do + if pat:match(found) then + return true + end + end + end + return false end +end - return fun.any(filter_func, task:get_text_parts()) +-- borrowed from mime_types.lua +-- ext is the last extension, LOWERCASED +-- ext2 is the one before last extension LOWERCASED +local function gen_extension(fname) + local filename_parts = rspamd_str_split(fname, '.') + local ext = {} + for n = 1, 2 do + ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil + end + return ext[1],ext[2],filename_parts end +local function check_parts_match(task, rule) + + local filter_func = function(p) + local content_type,content_subtype = p:get_type() + local fname = p:get_filename() + local ext, ext2, part_table + local extension_check = false + local content_type_check = false + local text_part_min_words_check = true + + if rule.scan_all_mime_parts == false then + -- check file extension and filename regex matching + if fname ~= nil then + ext,ext2,part_table = gen_extension(fname) + lua_util.debugm(rule.module_name, task, '%s: extension found: %s - 2.ext: %s - parts: %s', + rule.log_prefix, ext, ext2, part_table) + if match_filter(task, ext, rule.mime_parts_filter_ext) + or match_filter(task, ext2, rule.mime_parts_filter_ext) then + lua_util.debugm(rule.module_name, task, '%s: extension matched: %s', rule.log_prefix, ext) + extension_check = true + end + if match_filter(task, fname, rule.mime_parts_filter_regex) then + content_type_check = true + end + end + -- check content type regex matching + if content_type ~= nil and content_subtype ~= nil then + if match_filter(task, content_type..'/'..content_subtype, rule.mime_parts_filter_regex) then + lua_util.debugm(rule.module_name, task, '%s: regex ct: %s', rule.log_prefix, + content_type..'/'..content_subtype) + content_type_check = true + end + end + end + + -- check text_part has more words than text_part_min_words_check + if rule.text_part_min_words and p:is_text() then + text_part_min_words_check = p:get_words_count() >= tonumber(rule.text_part_min_words) + end + + return (rule.scan_image_mime and p:is_image()) + or (rule.scan_text_mime and text_part_min_words_check) + or (p:is_attachment() and rule.scan_all_mime_parts ~= false) + or extension_check + or content_type_check + end + + return fun.filter(filter_func, task:get_parts()) +end +exports.log_clean = log_clean exports.yield_result = yield_result exports.match_patterns = match_patterns exports.need_av_check = need_av_check exports.check_av_cache = check_av_cache exports.save_av_cache = save_av_cache -exports.text_parts_min_words = text_parts_min_words +exports.create_regex_table = create_regex_table +exports.check_parts_match = check_parts_match setmetatable(exports, { __call = function(t, override) @@ -210,4 +325,4 @@ setmetatable(exports, { end, }) -return exports
\ No newline at end of file +return exports |