}) | }) | ||||
end | end | ||||
if common.need_check(task, content, rule, digest) then | |||||
if common.need_check(task, content, rule, digest, clamav_check_uncached) then | |||||
return | |||||
else | |||||
clamav_check_uncached() | clamav_check_uncached() | ||||
end | end | ||||
end | end | ||||
end | end | ||||
local function check_cache(task, digest, rule, fn) | |||||
local function need_check(task, content, rule, digest, fn) | |||||
local uncached = true | |||||
local key = digest | local key = digest | ||||
local function redis_av_cb(err, data) | local function redis_av_cb(err, data) | ||||
if data and type(data) == 'string' then | if data and type(data) == 'string' then | ||||
-- Cached | -- Cached | ||||
data = rspamd_str_split(data, '\t') | |||||
local threat_string = rspamd_str_split(data[1], '\v') | |||||
data = lua_util.str_split(data, '\t') | |||||
local threat_string = lua_util.str_split(data[1], '\v') | |||||
local score = data[2] or rule.default_score | local score = data[2] or rule.default_score | ||||
if threat_string[1] ~= 'OK' then | if threat_string[1] ~= 'OK' then | ||||
lua_util.debugm(rule.name, task, '%s: got cached threat result for %s: %s - score: %s', | lua_util.debugm(rule.name, task, '%s: got cached threat result for %s: %s - score: %s', | ||||
lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s', | lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s', | ||||
rule.log_prefix, key, threat_string[1]) | rule.log_prefix, key, threat_string[1]) | ||||
end | end | ||||
uncached = false | |||||
else | else | ||||
if err then | if err then | ||||
rspamd_logger.errx(task, 'got error checking cache: %s', err) | rspamd_logger.errx(task, 'got error checking cache: %s', err) | ||||
end | end | ||||
return true | |||||
end | end | ||||
local f_message_not_too_large = message_not_too_large(task, content, rule) or true | |||||
local f_message_not_too_small = message_not_too_small(task, content, rule) or true | |||||
local f_message_min_words = message_min_words(task, rule) or true | |||||
local f_dynamic_scan = dynamic_scan(task, rule) or true | |||||
if uncached and | |||||
f_message_not_too_large and | |||||
f_message_not_too_small and | |||||
f_message_min_words and | |||||
f_dynamic_scan then | |||||
fn() | |||||
end | |||||
end | end | ||||
if rule.redis_params then | if rule.redis_params then | ||||
end | end | ||||
return false | return false | ||||
end | |||||
local function need_check(task, content, rule, digest) | |||||
return check_cache(task, digest, rule) and | |||||
message_not_too_large(task, content, rule) and | |||||
message_not_too_small(task, content, rule) and | |||||
message_min_words(task, rule) and | |||||
dynamic_scan(task, rule) | |||||
end | end | ||||
local function save_cache(task, digest, rule, to_save, dyn_weight) | local function save_cache(task, digest, rule, to_save, dyn_weight) | ||||
rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s', | rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s', | ||||
rule.detection_category, to_save, key, err) | rule.detection_category, to_save, key, err) | ||||
else | else | ||||
lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s', | |||||
rule.log_prefix, key, to_save, dyn_weight) | |||||
lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s - ttl %s', | |||||
rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire) | |||||
end | end | ||||
end | end | ||||
-- ext is the last extension, LOWERCASED | -- ext is the last extension, LOWERCASED | ||||
-- ext2 is the one before last extension LOWERCASED | -- ext2 is the one before last extension LOWERCASED | ||||
local function gen_extension(fname) | local function gen_extension(fname) | ||||
local filename_parts = rspamd_str_split(fname, '.') | |||||
local filename_parts = lua_util.str_split(fname, '.') | |||||
local ext = {} | local ext = {} | ||||
for n = 1, 2 do | for n = 1, 2 do |
}) | }) | ||||
end | end | ||||
if common.need_check(task, content, rule, digest) then | |||||
if common.need_check(task, content, rule, digest, dcc_check_uncached) then | |||||
return | |||||
else | |||||
dcc_check_uncached() | dcc_check_uncached() | ||||
end | end | ||||
}) | }) | ||||
end | end | ||||
if common.need_check(task, content, rule, digest) then | |||||
if common.need_check(task, content, rule, digest, fprot_check_uncached) then | |||||
return | |||||
else | |||||
fprot_check_uncached() | fprot_check_uncached() | ||||
end | end | ||||
local function icap_callback(err, conn) | local function icap_callback(err, conn) | ||||
local function icap_requery(error, info) | |||||
local function icap_requery(err_m, info) | |||||
-- set current upstream to fail because an error occurred | -- set current upstream to fail because an error occurred | ||||
upstream:fail() | upstream:fail() | ||||
lua_util.debugm(rule.name, task, | lua_util.debugm(rule.name, task, | ||||
'%s: %s Request Error: %s - retries left: %s', | '%s: %s Request Error: %s - retries left: %s', | ||||
rule.log_prefix, info, error, retransmits) | |||||
rule.log_prefix, info, err_m, retransmits) | |||||
-- Select a different upstream! | -- Select a different upstream! | ||||
upstream = rule.upstreams:get_upstream_round_robin() | upstream = rule.upstreams:get_upstream_round_robin() | ||||
}) | }) | ||||
else | else | ||||
rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. | rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. | ||||
'exceed - err: %s', rule.log_prefix, error) | |||||
common.yield_result(task, rule, 'failed - err: ' .. error, 0.0, 'fail') | |||||
'exceed - error: %s', rule.log_prefix, err_m or '') | |||||
common.yield_result(task, rule, 'failed - error: ' .. err_m or '', 0.0, 'fail') | |||||
end | end | ||||
end | end | ||||
local function get_respond_query() | local function get_respond_query() | ||||
table.insert(respond_headers, 1, | |||||
'RESPMOD icap://' .. addr:to_string() .. ':' .. addr:get_port() .. '/' | |||||
.. rule.scheme .. ' ICAP/1.0\r\n') | |||||
table.insert(respond_headers, 1, string.format( | |||||
'RESPMOD icap://%s:%s/%s ICAP/1.0\r\n', addr:to_string(), addr:get_port(), rule.scheme)) | |||||
table.insert(respond_headers, '\r\n') | table.insert(respond_headers, '\r\n') | ||||
table.insert(respond_headers, size .. '\r\n') | table.insert(respond_headers, size .. '\r\n') | ||||
table.insert(respond_headers, content) | table.insert(respond_headers, content) | ||||
end | end | ||||
local function add_respond_header(name, value) | local function add_respond_header(name, value) | ||||
table.insert(respond_headers, name .. ': ' .. value .. '\r\n' ) | |||||
if name and value then | |||||
table.insert(respond_headers, string.format('%s: %s\r\n', name, value)) | |||||
end | |||||
end | end | ||||
local function icap_result_header_table(result) | local function icap_result_header_table(result) | ||||
'%s: icap X-Virus-ID: %s', rule.log_prefix, icap_headers['X-Virus-ID']) | '%s: icap X-Virus-ID: %s', rule.log_prefix, icap_headers['X-Virus-ID']) | ||||
if string.find(icap_headers['X-Virus-ID'], ', ') then | if string.find(icap_headers['X-Virus-ID'], ', ') then | ||||
local vnames = lua_util.rspamd_str_split(string.gsub(icap_headers['X-Virus-ID'], "%s", ""), ',') or {} | |||||
local vnames = lua_util.str_split(string.gsub(icap_headers['X-Virus-ID'], "%s", ""), ',') or {} | |||||
for _,v in ipairs(vnames) do | for _,v in ipairs(vnames) do | ||||
table.insert(threat_string, v) | table.insert(threat_string, v) | ||||
rule.log_prefix, infection_name, infected_filename) | rule.log_prefix, infection_name, infected_filename) | ||||
if string.find(infection_name, ', ') then | if string.find(infection_name, ', ') then | ||||
local vnames = lua_util.rspamd_str_split(infection_name, ',') or {} | |||||
local vnames = lua_util.str_split(infection_name, ',') or {} | |||||
for _,v in ipairs(vnames) do | for _,v in ipairs(vnames) do | ||||
table.insert(threat_string, v) | table.insert(threat_string, v) | ||||
end | end | ||||
end | end | ||||
local function icap_r_respond_cb(error, data, connection) | |||||
if error or connection == nil then | |||||
icap_requery(err, "icap_r_respond_cb") | |||||
local function icap_r_respond_cb(err_m, data, connection) | |||||
if err_m or connection == nil then | |||||
icap_requery(err_m, "icap_r_respond_cb") | |||||
else | else | ||||
local result = tostring(data) | local result = tostring(data) | ||||
conn:close() | conn:close() | ||||
end | end | ||||
end | end | ||||
local function icap_w_respond_cb(error, connection) | |||||
if error or connection == nil then | |||||
icap_requery(err, "icap_w_respond_cb") | |||||
local function icap_w_respond_cb(err_m, connection) | |||||
if err_m or connection == nil then | |||||
icap_requery(err_m, "icap_w_respond_cb") | |||||
else | else | ||||
connection:add_read(icap_r_respond_cb, '\r\n\r\n') | connection:add_read(icap_r_respond_cb, '\r\n\r\n') | ||||
end | end | ||||
end | end | ||||
local function icap_r_options_cb(error, data, connection) | |||||
if error or connection == nil then | |||||
icap_requery(err, "icap_r_options_cb") | |||||
local function icap_r_options_cb(err_m, data, connection) | |||||
if err_m or connection == nil then | |||||
icap_requery(err_m, "icap_r_options_cb") | |||||
else | else | ||||
local icap_headers = icap_result_header_table(tostring(data)) | local icap_headers = icap_result_header_table(tostring(data)) | ||||
local from = task:get_from('mime') | local from = task:get_from('mime') | ||||
local rcpt_to = task:get_principal_recipient() | local rcpt_to = task:get_principal_recipient() | ||||
local client = task:get_from_ip() | local client = task:get_from_ip() | ||||
add_respond_header('X-Client-IP', client:to_string()) | |||||
if client then add_respond_header('X-Client-IP', client:to_string()) end | |||||
add_respond_header('X-Mail-From', from[1].addr) | add_respond_header('X-Mail-From', from[1].addr) | ||||
add_respond_header('X-Rcpt-To', rcpt_to) | add_respond_header('X-Rcpt-To', rcpt_to) | ||||
end | end | ||||
}) | }) | ||||
end | end | ||||
if common.need_check(task, content, rule, digest) then | |||||
if common.need_check(task, content, rule, digest, icap_check_uncached) then | |||||
return | |||||
else | |||||
icap_check_uncached() | icap_check_uncached() | ||||
end | end | ||||
}) | }) | ||||
end | end | ||||
if common.need_check(task, content, rule, digest) then | |||||
if common.need_check(task, content, rule, digest, kaspersky_check_uncached) then | |||||
return | |||||
else | |||||
kaspersky_check_uncached() | kaspersky_check_uncached() | ||||
end | end | ||||
end | end | ||||
if common.need_check(task, content, rule, digest) then | |||||
if common.need_check(task, content, rule, digest, oletools_check_uncached) then | |||||
return | |||||
else | |||||
oletools_check_uncached() | oletools_check_uncached() | ||||
end | end | ||||
}) | }) | ||||
end | end | ||||
if common.need_check(task, content, rule, digest) then | |||||
if common.need_check(task, content, rule, digest, savapi_check_uncached) then | |||||
return | |||||
else | |||||
savapi_check_uncached() | savapi_check_uncached() | ||||
end | end | ||||
}) | }) | ||||
end | end | ||||
if common.need_check(task, content, rule, digest) then | |||||
if common.need_check(task, content, rule, digest, sophos_check_uncached) then | |||||
return | |||||
else | |||||
sophos_check_uncached() | sophos_check_uncached() | ||||
end | end | ||||
common.save_cache(task, digest, rule, symbols, spam_score) | common.save_cache(task, digest, rule, symbols, spam_score) | ||||
else | else | ||||
local symbols_table = {} | local symbols_table = {} | ||||
symbols_table = rspamd_str_split(symbols, ",") | |||||
symbols_table = lua_util.str_split(symbols, ",") | |||||
lua_util.debugm(rule.N, task, '%s: returned symbols as table: %s', rule.log_prefix, symbols_table) | lua_util.debugm(rule.N, task, '%s: returned symbols as table: %s', rule.log_prefix, symbols_table) | ||||
common.yield_result(task, rule, symbols_table, spam_score) | common.yield_result(task, rule, symbols_table, spam_score) | ||||
}) | }) | ||||
end | end | ||||
if common.need_check(task, content, rule, digest) then | |||||
if common.need_check(task, content, rule, digest, spamassassin_check_uncached) then | |||||
return | |||||
else | |||||
spamassassin_check_uncached() | spamassassin_check_uncached() | ||||
end | end | ||||
end | end | ||||
local function vade_check(task, content, digest, rule) | local function vade_check(task, content, digest, rule) | ||||
local function vade_url(addr) | |||||
local url | |||||
if rule.use_https then | |||||
url = string.format('https://%s:%d%s', tostring(addr), | |||||
rule.default_port, rule.url) | |||||
else | |||||
url = string.format('http://%s:%d%s', tostring(addr), | |||||
rule.default_port, rule.url) | |||||
local function vade_check_uncached() | |||||
local function vade_url(addr) | |||||
local url | |||||
if rule.use_https then | |||||
url = string.format('https://%s:%d%s', tostring(addr), | |||||
rule.default_port, rule.url) | |||||
else | |||||
url = string.format('http://%s:%d%s', tostring(addr), | |||||
rule.default_port, rule.url) | |||||
end | |||||
return url | |||||
end | end | ||||
return url | |||||
end | |||||
local upstream = rule.upstreams:get_upstream_round_robin() | |||||
local addr = upstream:get_addr() | |||||
local retransmits = rule.retransmits | |||||
local upstream = rule.upstreams:get_upstream_round_robin() | |||||
local addr = upstream:get_addr() | |||||
local retransmits = rule.retransmits | |||||
local url = vade_url(addr) | |||||
local hdrs = {} | |||||
local url = vade_url(addr) | |||||
local hdrs = {} | |||||
local helo = task:get_helo() | |||||
if helo then | |||||
hdrs['X-Helo'] = helo | |||||
end | |||||
local mail_from = task:get_from('smtp') or {} | |||||
if mail_from[1] and #mail_from[1].addr > 1 then | |||||
hdrs['X-Mailfrom'] = mail_from[1].addr | |||||
end | |||||
local helo = task:get_helo() | |||||
if helo then | |||||
hdrs['X-Helo'] = helo | |||||
end | |||||
local mail_from = task:get_from('smtp') or {} | |||||
if mail_from[1] and #mail_from[1].addr > 1 then | |||||
hdrs['X-Mailfrom'] = mail_from[1].addr | |||||
end | |||||
local rcpt_to = task:get_recipients('smtp') | |||||
if rcpt_to then | |||||
hdrs['X-Rcptto'] = {} | |||||
for _, r in ipairs(rcpt_to) do | |||||
table.insert(hdrs['X-Rcptto'], r.addr) | |||||
end | |||||
end | |||||
local rcpt_to = task:get_recipients('smtp') | |||||
if rcpt_to then | |||||
hdrs['X-Rcptto'] = {} | |||||
for _, r in ipairs(rcpt_to) do | |||||
table.insert(hdrs['X-Rcptto'], r.addr) | |||||
local fip = task:get_from_ip() | |||||
if fip and fip:is_valid() then | |||||
hdrs['X-Inet'] = tostring(fip) | |||||
end | end | ||||
end | |||||
local fip = task:get_from_ip() | |||||
if fip and fip:is_valid() then | |||||
hdrs['X-Inet'] = tostring(fip) | |||||
end | |||||
local request_data = { | |||||
task = task, | |||||
url = url, | |||||
body = task:get_content(), | |||||
headers = hdrs, | |||||
timeout = rule.timeout, | |||||
} | |||||
local request_data = { | |||||
task = task, | |||||
url = url, | |||||
body = task:get_content(), | |||||
headers = hdrs, | |||||
timeout = rule.timeout, | |||||
} | |||||
local function vade_callback(http_err, code, body, headers) | |||||
local function vade_callback(http_err, code, body, headers) | |||||
local function vade_requery() | |||||
-- set current upstream to fail because an error occurred | |||||
upstream:fail() | |||||
local function vade_requery() | |||||
-- set current upstream to fail because an error occurred | |||||
upstream:fail() | |||||
-- retry with another upstream until retransmits exceeds | |||||
if retransmits > 0 then | |||||
-- retry with another upstream until retransmits exceeds | |||||
if retransmits > 0 then | |||||
retransmits = retransmits - 1 | |||||
retransmits = retransmits - 1 | |||||
lua_util.debugm(rule.name, task, | |||||
'%s: Request Error: %s - retries left: %s', | |||||
rule.log_prefix, http_err, retransmits) | |||||
lua_util.debugm(rule.name, task, | |||||
'%s: Request Error: %s - retries left: %s', | |||||
rule.log_prefix, http_err, retransmits) | |||||
-- Select a different upstream! | |||||
upstream = rule.upstreams:get_upstream_round_robin() | |||||
addr = upstream:get_addr() | |||||
url = vade_url(addr) | |||||
-- Select a different upstream! | |||||
upstream = rule.upstreams:get_upstream_round_robin() | |||||
addr = upstream:get_addr() | |||||
url = vade_url(addr) | |||||
lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', | |||||
rule.log_prefix, addr, addr:get_port()) | |||||
request_data.url = url | |||||
lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', | |||||
rule.log_prefix, addr, addr:get_port()) | |||||
request_data.url = url | |||||
http.request(request_data) | |||||
else | |||||
rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. | |||||
'exceed', rule.log_prefix) | |||||
task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '.. | |||||
'retransmits exceed') | |||||
end | |||||
end | |||||
http.request(request_data) | |||||
if http_err then | |||||
vade_requery() | |||||
else | else | ||||
rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. | |||||
'exceed', rule.log_prefix) | |||||
task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '.. | |||||
'retransmits exceed') | |||||
end | |||||
end | |||||
-- Parse the response | |||||
if upstream then upstream:ok() end | |||||
if code ~= 200 then | |||||
rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) | |||||
task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) | |||||
return | |||||
end | |||||
local parser = ucl.parser() | |||||
local ret, err = parser:parse_string(body) | |||||
if not ret then | |||||
rspamd_logger.errx(task, 'vade: bad response body (raw): %s', body) | |||||
task:insert_result(rule.symbol_fail, 1.0, 'Parser error: ' .. err) | |||||
return | |||||
end | |||||
local obj = parser:get_object() | |||||
local verdict = obj.verdict | |||||
if not verdict then | |||||
rspamd_logger.errx(task, 'vade: bad response JSON (no verdict): %s', obj) | |||||
task:insert_result(rule.symbol_fail, 1.0, 'No verdict/unknown verdict') | |||||
return | |||||
end | |||||
local vparts = lua_util.str_split(verdict, ":") | |||||
verdict = table.remove(vparts, 1) or verdict | |||||
if http_err then | |||||
vade_requery() | |||||
else | |||||
-- Parse the response | |||||
if upstream then upstream:ok() end | |||||
if code ~= 200 then | |||||
rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers) | |||||
task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code) | |||||
return | |||||
end | |||||
local parser = ucl.parser() | |||||
local ret, err = parser:parse_string(body) | |||||
if not ret then | |||||
rspamd_logger.errx(task, 'vade: bad response body (raw): %s', body) | |||||
task:insert_result(rule.symbol_fail, 1.0, 'Parser error: ' .. err) | |||||
return | |||||
end | |||||
local obj = parser:get_object() | |||||
local verdict = obj.verdict | |||||
if not verdict then | |||||
rspamd_logger.errx(task, 'vade: bad response JSON (no verdict): %s', obj) | |||||
task:insert_result(rule.symbol_fail, 1.0, 'No verdict/unknown verdict') | |||||
return | |||||
end | |||||
local vparts = lua_util.rspamd_str_split(verdict, ":") | |||||
verdict = table.remove(vparts, 1) or verdict | |||||
local sym = rule.symbols[verdict] | |||||
if not sym then | |||||
sym = rule.symbols.other | |||||
end | |||||
local sym = rule.symbols[verdict] | |||||
if not sym then | |||||
sym = rule.symbols.other | |||||
end | |||||
if not sym.symbol then | |||||
-- Subcategory match | |||||
local lvl = 'low' | |||||
if vparts and vparts[1] then | |||||
lvl = vparts[1] | |||||
end | |||||
if sym[lvl] then | |||||
sym = sym[lvl] | |||||
else | |||||
sym = rule.symbols.other | |||||
end | |||||
end | |||||
if not sym.symbol then | |||||
-- Subcategory match | |||||
local lvl = 'low' | |||||
if vparts and vparts[1] then | |||||
lvl = vparts[1] | |||||
local opts = {} | |||||
if obj.score then | |||||
table.insert(opts, 'score=' .. obj.score) | |||||
end | |||||
if obj.elapsed then | |||||
table.insert(opts, 'elapsed=' .. obj.elapsed) | |||||
end | end | ||||
if sym[lvl] then | |||||
sym = sym[lvl] | |||||
if rule.log_spamcause and obj.spamcause then | |||||
rspamd_logger.infox(task, 'vadesecure verdict="%s", score=%s, spamcause="%s", message-id="%s"', | |||||
verdict, obj.score, obj.spamcause, task:get_message_id()) | |||||
else | else | ||||
sym = rule.symbols.other | |||||
lua_util.debugm(rule.name, task, 'vadesecure returned verdict="%s", score=%s, spamcause="%s"', | |||||
verdict, obj.score, obj.spamcause) | |||||
end | end | ||||
end | |||||
local opts = {} | |||||
if obj.score then | |||||
table.insert(opts, 'score=' .. obj.score) | |||||
end | |||||
if obj.elapsed then | |||||
table.insert(opts, 'elapsed=' .. obj.elapsed) | |||||
end | |||||
if rule.log_spamcause and obj.spamcause then | |||||
rspamd_logger.infox(task, 'vadesecure verdict="%s", score=%s, spamcause="%s", message-id="%s"', | |||||
verdict, obj.score, obj.spamcause, task:get_message_id()) | |||||
else | |||||
lua_util.debugm(rule.name, task, 'vadesecure returned verdict="%s", score=%s, spamcause="%s"', | |||||
verdict, obj.score, obj.spamcause) | |||||
end | |||||
if #vparts > 0 then | |||||
table.insert(opts, 'verdict=' .. verdict .. ';' .. table.concat(vparts, ':')) | |||||
end | |||||
if #vparts > 0 then | |||||
table.insert(opts, 'verdict=' .. verdict .. ';' .. table.concat(vparts, ':')) | |||||
task:insert_result(sym.symbol, 1.0, opts) | |||||
end | end | ||||
task:insert_result(sym.symbol, 1.0, opts) | |||||
end | end | ||||
end | |||||
if common.need_check(task, content, rule, digest) then | |||||
request_data.callback = vade_callback | request_data.callback = vade_callback | ||||
http.request(request_data) | http.request(request_data) | ||||
end | end | ||||
if common.need_check(task, content, rule, digest, vade_check_uncached) then | |||||
return | |||||
else | |||||
vade_check_uncached() | |||||
end | |||||
end | end | ||||
return { | return { |