From 05d331d07ad6fdc791b750e182d5ca266e4c5aaa Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Thu, 17 Jan 2019 15:27:27 +0000 Subject: [PATCH] [Minor] Lua_scanners: Fix various issues --- lualib/lua_scanners/clamav.lua | 15 +++-- lualib/lua_scanners/common.lua | 12 ++-- lualib/lua_scanners/dcc.lua | 2 +- lualib/lua_scanners/fprot.lua | 2 +- lualib/lua_scanners/icap.lua | 28 ++++++---- lualib/lua_scanners/kaspersky_av.lua | 9 +-- lualib/lua_scanners/oletools.lua | 83 ++++++++++++++-------------- lualib/lua_scanners/savapi.lua | 16 +++--- lualib/lua_scanners/sophos.lua | 13 +++-- 9 files changed, 101 insertions(+), 79 deletions(-) diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua index 4ca3e8a8b..0f97028ea 100644 --- a/lualib/lua_scanners/clamav.lua +++ b/lualib/lua_scanners/clamav.lua @@ -32,7 +32,7 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"' local function clamav_config(opts) local clamav_conf = { - N = N, + name = N, scan_mime_parts = true, scan_text_mime = false, scan_image_mime = false, @@ -70,7 +70,7 @@ local function clamav_config(opts) clamav_conf.default_port) if clamav_conf['upstreams'] then - lua_util.add_debug_alias('antivirus', clamav_conf.N) + lua_util.add_debug_alias('antivirus', clamav_conf.name) return clamav_conf end @@ -103,7 +103,8 @@ local function clamav_check(task, content, digest, rule) upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() - lua_util.debugm(rule.N, task, '%s: retry IP: %s', rule.log_prefix, addr) + lua_util.debugm(rule.name, task, '%s: retry IP: %s', + rule.log_prefix, addr) tcp.request({ task = task, @@ -123,13 +124,15 @@ local function clamav_check(task, content, digest, rule) upstream:ok() data = tostring(data) local cached - lua_util.debugm(rule.N, task, '%s: got reply: %s', rule.log_prefix, data) + lua_util.debugm(rule.name, task, '%s: got reply: %s', + rule.log_prefix, data) if data == 'stream: OK' then cached = 'OK' if rule['log_clean'] then - rspamd_logger.infox(task, '%s: message or mime_part is clean', rule.log_prefix) + rspamd_logger.infox(task, '%s: message or mime_part is clean', + rule.log_prefix) else - lua_util.debugm(rule.N, task, '%s: message or mime_part is clean', rule.log_prefix) + lua_util.debugm(rule.name, task, '%s: message or mime_part is clean', rule.log_prefix) end else local vname = string.match(data, 'stream: (.+) FOUND') diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua index 0c76004eb..1696688db 100644 --- a/lualib/lua_scanners/common.lua +++ b/lualib/lua_scanners/common.lua @@ -61,17 +61,21 @@ local function match_patterns(default_sym, found, patterns, dyn_weight) end end -local function yield_result(task, rule, vname, N, dyn_weight) +local function yield_result(task, rule, vname, dyn_weight) local all_whitelisted = true if not dyn_weight then dyn_weight = 1.0 end if type(vname) == 'string' then - local symname, symscore = match_patterns(rule.symbol, vname, rule.patterns, dyn_weight) + local symname, symscore = match_patterns(rule.symbol, + vname, + rule.patterns, + dyn_weight) if rule.whitelist and rule.whitelist:get_key(vname) then rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule.log_prefix, vname) return end task:insert_result(symname, symscore, vname) - rspamd_logger.infox(task, '%s: %s found: "%s"', rule.log_prefix, rule.detection_category, vname) + rspamd_logger.infox(task, '%s: %s found: "%s"', rule.log_prefix, + rule.detection_category, vname) elseif type(vname) == 'table' then for _, vn in ipairs(vname) do local symname, symscore = match_patterns(rule.symbol, vn, rule.patterns, dyn_weight) @@ -94,7 +98,7 @@ local function yield_result(task, rule, vname, N, dyn_weight) lua_util.template(rule.message or 'Rejected', { SCANNER = rule.name, VIRUS = vname, - }), N) + }), rule.name) end end diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua index e5c0a1964..fcee609d7 100644 --- a/lualib/lua_scanners/dcc.lua +++ b/lualib/lua_scanners/dcc.lua @@ -276,7 +276,7 @@ local function dcc_config(opts) dcc_conf = lua_util.override_defaults(dcc_conf, opts) if not dcc_conf.prefix then - dcc_conf.prefix = 'rs_' .. dcc_conf.name .. '_' + dcc_conf.prefix = 'rs_' .. dcc_conf.N .. '_' end if not dcc_conf.log_prefix then diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua index 2004d8aa0..8be894126 100644 --- a/lualib/lua_scanners/fprot.lua +++ b/lualib/lua_scanners/fprot.lua @@ -31,7 +31,7 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"' local function fprot_config(opts) local fprot_conf = { - N = N, + name = N, scan_mime_parts = true, scan_text_mime = false, scan_image_mime = false, diff --git a/lualib/lua_scanners/icap.lua b/lualib/lua_scanners/icap.lua index 8810681f9..1e913211c 100644 --- a/lualib/lua_scanners/icap.lua +++ b/lualib/lua_scanners/icap.lua @@ -44,10 +44,12 @@ local function icap_check(task, content, digest, rule) "Encapsulated: null-body=0\r\n\r\n", } local size = string.format("%x", tonumber(#content)) - lua_util.debugm(rule.N, task, '%s: size: %s', rule.log_prefix, size) + lua_util.debugm(rule.name, task, '%s: size: %s', + rule.log_prefix, size) local function get_respond_query() - table.insert(respond_headers, 1, 'RESPMOD icap://' .. addr:to_string() .. ':' .. addr:get_port() .. '/' + table.insert(respond_headers, 1, + 'RESPMOD icap://' .. addr:to_string() .. ':' .. addr:get_port() .. '/' .. rule.scheme .. ' ICAP/1.0\r\n') table.insert(respond_headers, 'Encapsulated: res-body=0\r\n') table.insert(respond_headers, '\r\n') @@ -72,7 +74,8 @@ local function icap_check(task, content, digest, rule) icap_headers[key] = value end end - lua_util.debugm(rule.N, task, '%s: icap_headers: %s', rule.log_prefix, icap_headers) + lua_util.debugm(rule.name, task, '%s: icap_headers: %s', + rule.log_prefix, icap_headers) return icap_headers end @@ -99,10 +102,12 @@ local function icap_check(task, content, digest, rule) if icap_headers['X-Infection-Found'] ~= nil then pattern_symbols = "(Type%=%d; .* Threat%=)(.*)([;]+)" match = string.gsub(icap_headers['X-Infection-Found'], pattern_symbols, "%2") - lua_util.debugm(rule.N, task, '%s: icap X-Infection-Found: %s', rule.log_prefix, match) + lua_util.debugm(rule.name, task, + '%s: icap X-Infection-Found: %s', rule.log_prefix, match) table.insert(threat_string, match) elseif icap_headers['X-Virus-ID'] ~= nil then - lua_util.debugm(rule.N, task, '%s: icap X-Virus-ID: %s', rule.log_prefix, icap_headers['X-Virus-ID']) + lua_util.debugm(rule.name, task, + '%s: icap X-Virus-ID: %s', rule.log_prefix, icap_headers['X-Virus-ID']) table.insert(threat_string, icap_headers['X-Virus-ID']) end @@ -177,14 +182,15 @@ local function icap_check(task, content, digest, rule) retransmits = retransmits - 1 - lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s', - rule.log_prefix, error, retransmits) + lua_util.debugm(rule.name, task, + '%s: Request Error: %s - retries left: %s', + rule.log_prefix, error, retransmits) -- Select a different upstream! upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() - lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s', + lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', rule.log_prefix, addr, addr:get_port()) tcp.request({ @@ -237,7 +243,7 @@ end local function icap_config(opts) local icap_conf = { - N = N, + name = N, scan_mime_parts = true, scan_all_mime_parts = true, scan_text_mime = false, @@ -283,7 +289,7 @@ local function icap_config(opts) icap_conf.default_port) if icap_conf.upstreams then - lua_util.add_debug_alias('external_services', icap_conf.N) + lua_util.add_debug_alias('external_services', icap_conf.name) return icap_conf end @@ -293,7 +299,7 @@ local function icap_config(opts) end return { - type = {N,'virus', 'virus', 'scanner'}, + type = {N, 'virus', 'virus', 'scanner'}, description = 'generic icap antivirus', configure = icap_config, check = icap_check, diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua index f06e59cd7..ebed710de 100644 --- a/lualib/lua_scanners/kaspersky_av.lua +++ b/lualib/lua_scanners/kaspersky_av.lua @@ -32,7 +32,7 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"' local function kaspersky_config(opts) local kaspersky_conf = { - N = N, + name = N, scan_mime_parts = true, scan_text_mime = false, scan_image_mime = false, @@ -70,7 +70,7 @@ local function kaspersky_config(opts) kaspersky_conf['servers'], 0) if kaspersky_conf['upstreams'] then - lua_util.add_debug_alias('antivirus', kaspersky_conf.N) + lua_util.add_debug_alias('antivirus', kaspersky_conf.name) return kaspersky_conf end @@ -122,7 +122,7 @@ local function kaspersky_check(task, content, digest, rule) upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() - lua_util.debugm(rule.N, task, + lua_util.debugm(rule.name, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) tcp.request({ @@ -146,7 +146,8 @@ local function kaspersky_check(task, content, digest, rule) upstream:ok() data = tostring(data) local cached - lua_util.debugm(rule.N, task, '%s [%s]: got reply: %s', + lua_util.debugm(rule.name, task, + '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data) if data == 'stream: OK' or data == fname .. ': OK' then cached = 'OK' diff --git a/lualib/lua_scanners/oletools.lua b/lualib/lua_scanners/oletools.lua index 4ee5f040b..bd6cc9007 100644 --- a/lualib/lua_scanners/oletools.lua +++ b/lualib/lua_scanners/oletools.lua @@ -48,15 +48,16 @@ local function oletools_check(task, content, digest, rule) retransmits = retransmits - 1 - lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s', - rule.log_prefix, error, retransmits) + lua_util.debugm(rule.name, task, + '%s: Request Error: %s - retries left: %s', + rule.log_prefix, error, retransmits) -- Select a different upstream! upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() - lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s', - rule.log_prefix, addr, addr:get_port()) + lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s', + rule.log_prefix, addr, addr:get_port()) tcp.request({ task = task, @@ -69,7 +70,7 @@ local function oletools_check(task, content, digest, rule) }) else rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '.. - 'exceed - err: %s', rule.log_prefix, error) + 'exceed - err: %s', rule.log_prefix, error) task:insert_result(rule.symbol_fail, 0.0, 'failed - err: ' .. error) end end @@ -87,9 +88,9 @@ local function oletools_check(task, content, digest, rule) local ucl_parser = ucl.parser() local ok, ucl_err = ucl_parser:parse_string(tostring(data)) if not ok then - rspamd_logger.errx(task, "%s: error parsing json response: %s", + rspamd_logger.errx(task, "%s: error parsing json response: %s", rule.log_prefix, ucl_err) - return + return end local result = ucl_parser:get_object() @@ -109,24 +110,24 @@ local function oletools_check(task, content, digest, rule) if result[1].error ~= nil then rspamd_logger.errx(task, '%s: ERROR found: %s', rule.log_prefix, - result[1].error) - if result[1].error == 'File too small' then - common.save_av_cache(task, digest, rule, 'OK') - common.log_clean(task, rule, 'File too small to be scanned for macros') - else - oletools_requery(result[1].error) - end + result[1].error) + if result[1].error == 'File too small' then + common.save_av_cache(task, digest, rule, 'OK') + common.log_clean(task, rule, 'File too small to be scanned for macros') + else + oletools_requery(result[1].error) + end elseif result[3]['return_code'] == 9 then rspamd_logger.warnx(task, '%s: File is encrypted.', rule.log_prefix) elseif result[3]['return_code'] > 6 then rspamd_logger.errx(task, '%s: Error Returned: %s', - rule.log_prefix, oletools_rc[result[3]['return_code']]) + rule.log_prefix, oletools_rc[result[3]['return_code']]) rspamd_logger.errx(task, '%s: Error message: %s', - rule.log_prefix, result[2]['message']) + rule.log_prefix, result[2]['message']) task:insert_result(rule.symbol_fail, 0.0, 'failed - err: ' .. oletools_rc[result[3]['return_code']]) elseif result[3]['return_code'] > 1 then rspamd_logger.errx(task, '%s: Error message: %s', - rule.log_prefix, result[2]['message']) + rule.log_prefix, result[2]['message']) oletools_requery(oletools_rc[result[3]['return_code']]) elseif #result[2]['analysis'] == 0 and #result[2]['macros'] == 0 then rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix) @@ -146,19 +147,21 @@ local function oletools_check(task, content, digest, rule) local m_dridex = '-' local m_vba = '-' - lua_util.debugm(rule.N, task, '%s: filename: %s', rule.log_prefix, result[2]['file']) - lua_util.debugm(rule.N, task, '%s: type: %s', rule.log_prefix, result[2]['type']) + lua_util.debugm(rule.name, task, + '%s: filename: %s', rule.log_prefix, result[2]['file']) + lua_util.debugm(rule.name, task, + '%s: type: %s', rule.log_prefix, result[2]['type']) for _,m in ipairs(result[2]['macros']) do - lua_util.debugm(rule.N, task, '%s: macros found - code: %s, ole_stream: %s, '.. - 'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename) + lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, '.. + 'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename) end local analysis_keyword_table = {} for _,a in ipairs(result[2]['analysis']) do - lua_util.debugm(rule.N, task, '%s: threat found - type: %s, keyword: %s, '.. - 'description: %s', rule.log_prefix, a.type, a.keyword, a.description) + lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, '.. + 'description: %s', rule.log_prefix, a.type, a.keyword, a.description) if a.type == 'AutoExec' then m_autoexec = 'A' table.insert(analysis_keyword_table, a.keyword) @@ -181,12 +184,12 @@ local function oletools_check(task, content, digest, rule) end end - --lua_util.debugm(N, task, '%s: analysis_keyword_table: %s', rule.log_prefix, analysis_keyword_table) + --lua_util.debugm(N, task, '%s: analysis_keyword_table: %s', rule.log_prefix, analysis_keyword_table) if rule.extended == false and m_autoexec == 'A' and m_suspicious == 'S' then -- use single string as virus name local threat = 'AutoExec + Suspicious (' .. table.concat(analysis_keyword_table, ',') .. ')' - lua_util.debugm(rule.N, task, '%s: threat result: %s', rule.log_prefix, threat) + lua_util.debugm(rule.name, task, '%s: threat result: %s', rule.log_prefix, threat) common.yield_result(task, rule, threat, rule.default_score) common.save_av_cache(task, digest, rule, threat, rule.default_score) @@ -194,17 +197,17 @@ local function oletools_check(task, content, digest, rule) -- report any flags (types) and any most keywords as individual virus name local flags = m_exist .. - m_autoexec .. - m_suspicious .. - m_iocs .. - m_hex .. - m_base64 .. - m_dridex .. - m_vba + m_autoexec .. + m_suspicious .. + m_iocs .. + m_hex .. + m_base64 .. + m_dridex .. + m_vba table.insert(analysis_keyword_table, 1, flags) - lua_util.debugm(rule.N, task, '%s: extended threat result: %s', - rule.log_prefix, table.concat(analysis_keyword_table, ',')) + lua_util.debugm(rule.name, task, '%s: extended threat result: %s', + rule.log_prefix, table.concat(analysis_keyword_table, ',')) common.yield_result(task, rule, analysis_keyword_table, rule.default_score) common.save_av_cache(task, digest, rule, analysis_keyword_table, rule.default_score) @@ -243,7 +246,7 @@ end local function oletools_config(opts) local oletools_conf = { - N = N, + name = N, scan_mime_parts = false, scan_text_mime = false, scan_image_mime = false, @@ -280,21 +283,21 @@ local function oletools_config(opts) end oletools_conf.upstreams = upstream_list.create(rspamd_config, - oletools_conf.servers, - oletools_conf.default_port) + oletools_conf.servers, + oletools_conf.default_port) if oletools_conf.upstreams then - lua_util.add_debug_alias('external_services', oletools_conf.N) + lua_util.add_debug_alias('external_services', oletools_conf.name) return oletools_conf end rspamd_logger.errx(rspamd_config, 'cannot parse servers %s', - oletools_conf.servers) + oletools_conf.servers) return nil end return { - type = {N,'attachment scanner', 'hash', 'scanner'}, + type = {N, 'attachment scanner', 'hash', 'scanner'}, description = 'oletools office macro scanner', configure = oletools_config, check = oletools_check, diff --git a/lualib/lua_scanners/savapi.lua b/lualib/lua_scanners/savapi.lua index 1393cd027..4a7b7082a 100644 --- a/lualib/lua_scanners/savapi.lua +++ b/lualib/lua_scanners/savapi.lua @@ -32,7 +32,7 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"' local function savapi_config(opts) local savapi_conf = { - N = N, + name = N, scan_mime_parts = true, scan_text_mime = false, scan_image_mime = false, @@ -72,7 +72,7 @@ local function savapi_config(opts) savapi_conf.default_port) if savapi_conf['upstreams'] then - lua_util.add_debug_alias('antivirus', savapi_conf.N) + lua_util.add_debug_alias('antivirus', savapi_conf.name) return savapi_conf end @@ -119,7 +119,7 @@ local function savapi_check(task, content, digest, rule) for virus,_ in pairs(vnames) do table.insert(vnames_reordered, virus) end - lua_util.debugm(rule.N, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered) + lua_util.debugm(rule.name, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered) if #vnames_reordered > 0 then local vname = {} for _,virus in ipairs(vnames_reordered) do @@ -136,8 +136,8 @@ local function savapi_check(task, content, digest, rule) local function savapi_scan2_cb(err, data, conn) local result = tostring(data) - lua_util.debugm(rule.N, task, "%s: got reply: %s", - rule['type'], result) + lua_util.debugm(rule.name, task, "%s: got reply: %s", + rule.type, result) -- Terminal response - clean if string.find(result, '200') or string.find(result, '210') then @@ -178,7 +178,7 @@ local function savapi_check(task, content, digest, rule) local function savapi_greet2_cb(err, data, conn) local result = tostring(data) if string.find(result, '100 PRODUCT') then - lua_util.debugm(rule.N, task, "%s: scanning file: %s", + lua_util.debugm(rule.name, task, "%s: scanning file: %s", rule['type'], fname) conn:add_write(savapi_scan1_cb, {string.format('SCAN %s\n', fname)}) @@ -208,7 +208,9 @@ local function savapi_check(task, content, digest, rule) upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() - lua_util.debugm(rule.N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + lua_util.debugm(rule.name, task, + '%s [%s]: retry IP: %s', rule['symbol'], + rule['type'], addr) tcp.request({ task = task, diff --git a/lualib/lua_scanners/sophos.lua b/lualib/lua_scanners/sophos.lua index 3919d9449..934ce1f79 100644 --- a/lualib/lua_scanners/sophos.lua +++ b/lualib/lua_scanners/sophos.lua @@ -31,7 +31,7 @@ local default_message = '${SCANNER}: virus found: "${VIRUS}"' local function sophos_config(opts) local sophos_conf = { - N = N, + name = N, scan_mime_parts = true, scan_text_mime = false, scan_image_mime = false, @@ -71,7 +71,7 @@ local function sophos_config(opts) sophos_conf.default_port) if sophos_conf['upstreams'] then - lua_util.add_debug_alias('antivirus', sophos_conf.N) + lua_util.add_debug_alias('antivirus', sophos_conf.name) return sophos_conf end @@ -104,7 +104,8 @@ local function sophos_check(task, content, digest, rule) upstream = rule.upstreams:get_upstream_round_robin() addr = upstream:get_addr() - lua_util.debugm(rule.N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) + lua_util.debugm(rule.name, task, + '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr) tcp.request({ task = task, @@ -121,7 +122,8 @@ local function sophos_check(task, content, digest, rule) else upstream:ok() data = tostring(data) - lua_util.debugm(rule.N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data) + lua_util.debugm(rule.name, task, + '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data) local vname = string.match(data, 'VIRUS (%S+) ') if vname then common.yield_result(task, rule, vname) @@ -131,7 +133,8 @@ local function sophos_check(task, content, digest, rule) if rule['log_clean'] then rspamd_logger.infox(task, '%s: message or mime_part is clean', rule.log_prefix) else - lua_util.debugm(rule.N, task, '%s: message or mime_part is clean', rule.log_prefix) + lua_util.debugm(rule.name, task, + '%s: message or mime_part is clean', rule.log_prefix) end common.save_av_cache(task, digest, rule, 'OK') -- not finished - continue -- 2.39.5